From a20960834bb019b17431c30d7cfd174c14792113 Mon Sep 17 00:00:00 2001 From: mcy Date: Thu, 31 Jan 2019 09:45:52 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E8=BF=9C=E7=A8=8B=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../monitor/remote/DbRemoteConfigLoader.java | 79 +------------------ .../remote/RemoteAdapterMonitorImpl.java | 56 +++++++++++++ .../otter/canal/common/utils/CommonUtils.java | 53 +++++++++++++ .../monitor/remote/DbRemoteConfigLoader.java | 73 +---------------- .../remote/RemoteInstanceMonitorImpl.java | 55 +++++++++++++ 5 files changed, 168 insertions(+), 148 deletions(-) create mode 100644 client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java index 704a606f..a9a3bb6e 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java @@ -1,6 +1,5 @@ package com.alibaba.otter.canal.adapter.launcher.monitor.remote; -import java.io.File; import java.io.FileWriter; import java.sql.Connection; import java.sql.ResultSet; @@ -19,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.common.utils.CommonUtils; import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.google.common.base.Joiner; import com.google.common.collect.MapMaker; @@ -115,7 +115,7 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader { * @param content 文件内容 */ private void overrideLocalCanalConfig(String content) { - try (FileWriter writer = new FileWriter(getConfPath() + "application.yml")) { + try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) { writer.write(content); writer.flush(); } catch (Exception e) { @@ -209,41 +209,6 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader { } } - private static boolean deleteDir(File dirFile) { - if (!dirFile.exists()) { - return false; - } - - if (dirFile.isFile()) { - return dirFile.delete(); - } else { - File[] files = dirFile.listFiles(); - if (files == null || files.length == 0) { - return dirFile.delete(); - } - for (File file : files) { - deleteDir(file); - } - } - - return dirFile.delete(); - } - - /** - * 获取conf文件夹所在路径 - * - * @return 路径地址 - */ - private String getConfPath() { - String classpath = this.getClass().getResource("/").getPath(); - String confPath = classpath + "../conf/"; - if (new File(confPath).exists()) { - return confPath; - } else { - return classpath; - } - } - /** * 启动监听数据库变化 */ @@ -280,44 +245,4 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader { logger.error(e.getMessage(), e); } } - - private class RemoteAdapterMonitorImpl implements RemoteAdapterMonitor { - - @Override - public void onAdd(ConfigItem configItem) { - this.onModify(configItem); - } - - @Override - public void onModify(ConfigItem configItem) { - String confPath = getConfPath(); - String category = configItem.getCategory(); - File categoryDir = new File(confPath + category); - if (!categoryDir.isDirectory()) { - boolean mkDirs = categoryDir.mkdirs(); - if (!mkDirs) { - logger.info("## Create adapter category dir error: {}", category); - return; - } - } - String name = configItem.getName(); - try (FileWriter writer = new FileWriter( - confPath + category + "/" + configItem.getName())) { - writer.write(configItem.getContent()); - writer.flush(); - logger.info("## Loaded remote adapter config: {}/{}", category, name); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public void onDelete(String name) { - File file = new File(getConfPath() + name); - if (file.exists()) { - deleteDir(file); - logger.info("## Deleted and reloaded remote adapter config: {}", name); - } - } - } } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java new file mode 100644 index 00000000..6b4c2b81 --- /dev/null +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java @@ -0,0 +1,56 @@ +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +import com.alibaba.otter.canal.common.utils.CommonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; + +/** + * 远程配置监听器实现 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class RemoteAdapterMonitorImpl implements RemoteAdapterMonitor { + + private static final Logger logger = LoggerFactory.getLogger(RemoteAdapterMonitorImpl.class); + + @Override + public void onAdd(ConfigItem configItem) { + this.onModify(configItem); + } + + @Override + public void onModify(ConfigItem configItem) { + String confPath = CommonUtils.getConfPath(); + String category = configItem.getCategory(); + File categoryDir = new File(confPath + category); + if (!categoryDir.isDirectory()) { + boolean mkDirs = categoryDir.mkdirs(); + if (!mkDirs) { + logger.info("## Create adapter category dir error: {}", category); + return; + } + } + String name = configItem.getName(); + try (FileWriter writer = new FileWriter(confPath + category + "/" + configItem.getName())) { + writer.write(configItem.getContent()); + writer.flush(); + logger.info("## Loaded remote adapter config: {}/{}", category, name); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onDelete(String name) { + File file = new File(CommonUtils.getConfPath() + name); + if (file.exists()) { + CommonUtils.deleteDir(file); + logger.info("## Deleted and reloaded remote adapter config: {}", name); + } + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java new file mode 100644 index 00000000..52b9621a --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/CommonUtils.java @@ -0,0 +1,53 @@ +package com.alibaba.otter.canal.common.utils; + +import java.io.File; + +/** + * 通用工具类 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class CommonUtils { + + /** + * 获取conf文件夹所在路径 + * + * @return 路径地址 + */ + public static String getConfPath() { + String classpath = CommonUtils.class.getResource("/").getPath(); + String confPath = classpath + "../conf/"; + if (new File(confPath).exists()) { + return confPath; + } else { + return classpath; + } + } + + /** + * 删除文件夹 + * + * @param dirFile 文件夹对象 + * @return 是否删除成功 + */ + public static boolean deleteDir(File dirFile) { + if (!dirFile.exists()) { + return false; + } + + if (dirFile.isFile()) { + return dirFile.delete(); + } else { + File[] files = dirFile.listFiles(); + if (files == null || files.length == 0) { + return dirFile.delete(); + } + for (File file : files) { + deleteDir(file); + } + } + + return dirFile.delete(); + } +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java index 7cf7fef0..6cec0e46 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/DbRemoteConfigLoader.java @@ -1,7 +1,6 @@ package com.alibaba.otter.canal.deployer.monitor.remote; import java.io.ByteArrayInputStream; -import java.io.File; import java.io.FileWriter; import java.nio.charset.StandardCharsets; import java.sql.Connection; @@ -18,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.common.utils.CommonUtils; import com.alibaba.otter.canal.common.utils.NamedThreadFactory; import com.alibaba.otter.canal.deployer.CanalConstants; import com.google.common.base.Joiner; @@ -101,7 +101,7 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader { * @param content 远程配置内容文本 */ private void overrideLocalCanalConfig(String content) { - try (FileWriter writer = new FileWriter(getConfPath() + "canal.properties")) { + try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "canal.properties")) { writer.write(content); writer.flush(); } catch (Exception e) { @@ -213,26 +213,6 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader { } } - private static boolean deleteDir(File dirFile) { - if (!dirFile.exists()) { - return false; - } - - if (dirFile.isFile()) { - return dirFile.delete(); - } else { - File[] files = dirFile.listFiles(); - if (files == null || files.length == 0) { - return dirFile.delete(); - } - for (File file : files) { - deleteDir(file); - } - } - - return dirFile.delete(); - } - /** * 监听 canal 主配置和 instance 配置变化 * @@ -281,53 +261,4 @@ public class DbRemoteConfigLoader implements RemoteConfigLoader { } } - /** - * 获取conf文件夹所在路径 - * - * @return 路径地址 - */ - private String getConfPath() { - String classpath = this.getClass().getResource("/").getPath(); - String confPath = classpath + ".." + File.separator + "conf" + File.separator; - if (new File(confPath).exists()) { - return confPath; - } else { - return classpath; - } - } - - /** - * 远程xxx/instance.properties配置监听器实现 - */ - private class RemoteInstanceMonitorImpl implements RemoteInstanceMonitor { - - @Override - public void onAdd(ConfigItem configItem) { - this.onModify(configItem); - } - - @Override - public void onModify(ConfigItem configItem) { - File instanceDir = new File(getConfPath() + configItem.getName()); - if (!instanceDir.exists()) { - instanceDir.mkdirs(); - } - try (FileWriter writer = new FileWriter(getConfPath() + configItem.getName() + "/instance.properties")) { - writer.write(configItem.getContent()); - writer.flush(); - logger.info("## Loaded remote instance config: {}/instance.properties ", configItem.getName()); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public void onDelete(String instanceName) { - File file = new File(getConfPath() + instanceName + "/"); - if (file.exists()) { - deleteDir(file); - logger.info("## Deleted and loaded remote instance config: {} ", instanceName); - } - } - } } diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java new file mode 100644 index 00000000..8eded2b1 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/remote/RemoteInstanceMonitorImpl.java @@ -0,0 +1,55 @@ +package com.alibaba.otter.canal.deployer.monitor.remote; + +import java.io.File; +import java.io.FileWriter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.common.utils.CommonUtils; + +/** + * 远程xxx/instance.properties配置监听器实现 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class RemoteInstanceMonitorImpl implements RemoteInstanceMonitor { + + private static final Logger logger = LoggerFactory.getLogger(RemoteInstanceMonitorImpl.class); + + @Override + public void onAdd(ConfigItem configItem) { + this.onModify(configItem); + } + + @Override + public void onModify(ConfigItem configItem) { + String confDir = CommonUtils.getConfPath() + configItem.getName(); + File instanceDir = new File(confDir); + if (!instanceDir.exists()) { + boolean mkDirs = instanceDir.mkdirs(); + if (!mkDirs) { + logger.info("## Error to create instance config dir: {}", configItem.getName()); + return; + } + } + try (FileWriter writer = new FileWriter(confDir + "/instance.properties")) { + writer.write(configItem.getContent()); + writer.flush(); + logger.info("## Loaded remote instance config: {}/instance.properties ", configItem.getName()); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onDelete(String instanceName) { + File file = new File(CommonUtils.getConfPath() + instanceName + "/"); + if (file.exists()) { + CommonUtils.deleteDir(file); + logger.info("## Deleted and loaded remote instance config: {} ", instanceName); + } + } + +} From 8211ef35e62b166c630f4e9224f675fe6b8aabdc Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 1 Feb 2019 10:11:52 +0800 Subject: [PATCH 02/10] =?UTF-8?q?=E4=BF=AE=E6=94=B9yml=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E7=BB=91=E5=AE=9A=EF=BC=8C=E9=81=B5=E5=BE=AA?= =?UTF-8?q?spring=20yml=E5=8A=A0=E8=BD=BD=E6=96=B9=E5=BC=8F=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81environment=E7=BA=A7=E5=8D=A0=E4=BD=8D?= =?UTF-8?q?=E7=AC=A6=E6=9B=BF=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client-adapter/common/pom.xml | 13 +- .../canal/client/adapter/OuterAdapter.java | 10 +- .../adapter/config/YmlConfigBinder.java | 148 ++++ .../DefaultPropertyNamePatternsMatcher.java | 97 +++ .../config/bind/InetAddressEditor.java | 31 + .../bind/OriginCapablePropertyValue.java | 52 ++ .../PatternPropertyNamePatternsMatcher.java | 27 + .../bind/PropertiesConfigurationFactory.java | 356 +++++++++ .../bind/PropertyNamePatternsMatcher.java | 38 + .../adapter/config/bind/PropertyOrigin.java | 30 + .../bind/PropertySourcesPropertyResolver.java | 164 ++++ .../bind/PropertySourcesPropertyValues.java | 233 ++++++ .../config/bind/RelaxedConversionService.java | 127 +++ .../config/bind/RelaxedDataBinder.java | 729 ++++++++++++++++++ .../adapter/config/bind/RelaxedNames.java | 241 ++++++ .../bind/StringToCharArrayConverter.java | 17 + .../config/common/AbstractResource.java | 203 +++++ .../config/common/ByteArrayResource.java | 118 +++ .../common/CompositePropertySource.java | 107 +++ .../common/EnumerablePropertySource.java | 58 ++ .../config/common/MapPropertySource.java | 38 + .../config/common/MutablePropertySources.java | 221 ++++++ .../common/PropertiesPropertySource.java | 34 + .../adapter/config/common/PropertySource.java | 239 ++++++ .../config/common/PropertySourceLoader.java | 35 + .../config/common/PropertySources.java | 25 + .../adapter/config/common/Resource.java | 57 ++ .../common/SpringProfileDocumentMatcher.java | 182 +++++ .../adapter/config/common/YamlProcessor.java | 419 ++++++++++ .../common/YamlPropertySourceLoader.java | 87 +++ client-adapter/elasticsearch/pom.xml | 8 +- .../canal/client/adapter/es/ESAdapter.java | 15 +- .../adapter/es/config/ESSyncConfigLoader.java | 12 +- .../adapter/es/test/ConfigLoadTest.java | 2 +- .../client/adapter/es/test/sync/Common.java | 2 +- client-adapter/hbase/pom.xml | 6 - .../client/adapter/hbase/HbaseAdapter.java | 9 +- .../hbase/config/MappingConfigLoader.java | 12 +- .../launcher/loader/CanalAdapterLoader.java | 44 +- .../adapter/logger/LoggerAdapterExample.java | 3 +- client-adapter/rdb/pom.xml | 7 - .../canal/client/adapter/rdb/RdbAdapter.java | 15 +- .../adapter/rdb/config/ConfigLoader.java | 10 +- .../adapter/rdb/test/ConfigLoadTest.java | 2 +- .../client/adapter/rdb/test/sync/Common.java | 2 +- 45 files changed, 4195 insertions(+), 90 deletions(-) create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java create mode 100644 client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index 9c97646b..ec3ab9b8 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -1,5 +1,6 @@ - + canal.client-adapter com.alibaba.otter @@ -26,6 +27,16 @@ druid 1.1.9 + + org.springframework + spring-context + 5.0.5.RELEASE + + + org.yaml + snakeyaml + 1.19 + diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index 37786222..31004f16 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter; import java.util.List; import java.util.Map; +import java.util.Properties; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.EtlResult; @@ -21,8 +22,9 @@ public interface OuterAdapter { * 外部适配器初始化接口 * * @param configuration 外部适配器配置信息 + * @param envProperties 环境变量的配置属性 */ - void init(OuterAdapterConfig configuration); + void init(OuterAdapterConfig configuration, Properties envProperties); /** * 往适配器中同步数据 @@ -38,7 +40,7 @@ public interface OuterAdapter { /** * Etl操作 - * + * * @param task 任务名, 对应配置名 * @param params etl筛选条件 */ @@ -48,7 +50,7 @@ public interface OuterAdapter { /** * 计算总数 - * + * * @param task 任务名, 对应配置名 * @return 总数 */ @@ -58,7 +60,7 @@ public interface OuterAdapter { /** * 通过task获取对应的destination - * + * * @param task 任务名, 对应配置名 * @return destination */ diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java new file mode 100644 index 00000000..8d49e726 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java @@ -0,0 +1,148 @@ +package com.alibaba.otter.canal.client.adapter.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.springframework.util.PropertyPlaceholderHelper; +import org.springframework.util.StringUtils; + +import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; +import com.alibaba.otter.canal.client.adapter.config.common.*; + +/** + * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 + * + * @author reweerma 2019-2-1 上午9:14:02 + * @version 1.0.0 + */ +public class YmlConfigBinder { + + /** + * 将当前内容绑定到指定对象 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz) { + return bindYmlToObj(null, content, clazz, null); + } + + /** + * 将当前内容绑定到指定对象并指定内容编码格式 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz, String charset) { + return bindYmlToObj(null, content, clazz, charset); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz) { + return bindYmlToObj(prefix, content, clazz, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset) { + return bindYmlToObj(prefix, content, clazz, charset, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: + * 当前内容有属性 zkServers: ${zookeeper.servers} + * 在envProperties中有属性 zookeeper.servers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 + * 则当前内容 zkServers 会被替换为 zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 + * 注: 假设绑定的类中 zkServers 属性是 List 对象, 则会自动映射成List + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset, + Properties baseProperties) { + try { + byte[] contentBytes; + if (charset == null) { + contentBytes = content.getBytes(); + } else { + contentBytes = content.getBytes(charset); + } + YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); + Resource configResource = new ByteArrayResource(contentBytes); + PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); + + Properties properties = new Properties(); + Map propertiesRes = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { + prefix = prefix + "."; + } + + properties.putAll((Map) propertySource.getSource()); + + if (baseProperties != null) { + baseProperties.putAll(properties); + properties = baseProperties; + } + + for (Object o : ((Map) propertySource.getSource()).entrySet()) { + Map.Entry entry = (Map.Entry) o; + String key = (String) entry.getKey(); + Object value = entry.getValue(); + + if (prefix != null) { + if (key != null && key.startsWith(prefix)) { + key = key.substring(prefix.length()); + } else { + continue; + } + } + + if (value != null && value.toString().contains("${")) { + PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); + value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties); + } + + propertiesRes.put(key, value); + } + + propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); + + T target = clazz.newInstance(); + + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(propertySource); + + PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory(target); + factory.setPropertySources(propertySources); + factory.setIgnoreInvalidFields(true); + factory.setIgnoreUnknownFields(true); + + factory.bindPropertiesToTarget(); + + return target; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..0b63ed2c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java @@ -0,0 +1,97 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final char[] delimiters; + + private final boolean ignoreCase; + + private final String[] names; + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){ + this(delimiters, false, names); + } + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){ + this(delimiters, ignoreCase, new HashSet(Arrays.asList(names))); + } + + DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set names){ + this.delimiters = delimiters; + this.ignoreCase = ignoreCase; + this.names = names.toArray(new String[names.size()]); + } + + @Override + public boolean matches(String propertyName) { + char[] propertyNameChars = propertyName.toCharArray(); + boolean[] match = new boolean[this.names.length]; + boolean noneMatched = true; + for (int i = 0; i < this.names.length; i++) { + if (this.names[i].length() <= propertyNameChars.length) { + match[i] = true; + noneMatched = false; + } + } + if (noneMatched) { + return false; + } + for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) { + for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) { + if (match[nameIndex]) { + match[nameIndex] = false; + if (charIndex < this.names[nameIndex].length()) { + if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) { + match[nameIndex] = true; + noneMatched = false; + } + } else { + char charAfter = propertyNameChars[this.names[nameIndex].length()]; + if (isDelimiter(charAfter)) { + match[nameIndex] = true; + noneMatched = false; + } + } + } + } + if (noneMatched) { + return false; + } + } + for (int i = 0; i < match.length; i++) { + if (match[i]) { + return true; + } + } + return false; + } + + private boolean isCharMatch(char c1, char c2) { + if (this.ignoreCase) { + return Character.toLowerCase(c1) == Character.toLowerCase(c2); + } + return c1 == c2; + } + + private boolean isDelimiter(char c) { + for (char delimiter : this.delimiters) { + if (c == delimiter) { + return true; + } + } + return false; + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java new file mode 100644 index 00000000..c9b259e3 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditorSupport; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class InetAddressEditor extends PropertyEditorSupport { + + @Override + public String getAsText() { + return ((InetAddress) getValue()).getHostAddress(); + } + + @Override + public void setAsText(String text) throws IllegalArgumentException { + try { + setValue(InetAddress.getByName(text)); + } catch (UnknownHostException ex) { + throw new IllegalArgumentException("Cannot locate host", ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java new file mode 100644 index 00000000..184d3d0c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java @@ -0,0 +1,52 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.beans.PropertyValue; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * A {@link PropertyValue} that can provide information about its origin. + * + * @author Andy Wilkinson + */ +class OriginCapablePropertyValue extends PropertyValue { + + private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; + + private final PropertyOrigin origin; + + OriginCapablePropertyValue(PropertyValue propertyValue){ + this(propertyValue.getName(), + propertyValue.getValue(), + (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); + } + + OriginCapablePropertyValue(String name, Object value, String originName, PropertySource originSource){ + this(name, value, new PropertyOrigin(originSource, originName)); + } + + OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ + super(name, value); + this.origin = origin; + setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); + } + + public PropertyOrigin getOrigin() { + return this.origin; + } + + @Override + public String toString() { + String name = (this.origin != null ? this.origin.getName() : this.getName()); + String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); + return "'" + name + "' from '" + source + "'"; + } + + public static PropertyOrigin getOrigin(PropertyValue propertyValue) { + if (propertyValue instanceof OriginCapablePropertyValue) { + return ((OriginCapablePropertyValue) propertyValue).getOrigin(); + } + return new OriginCapablePropertyValue(propertyValue).getOrigin(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..a041a673 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; + +import org.springframework.util.PatternMatchUtils; + +/** + * {@link PropertyNamePatternsMatcher} that delegates to + * {@link PatternMatchUtils#simpleMatch(String[], String)}. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final String[] patterns; + + PatternPropertyNamePatternsMatcher(Collection patterns){ + this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {}); + } + + @Override + public boolean matches(String propertyName) { + return PatternMatchUtils.simpleMatch(this.patterns, propertyName); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java new file mode 100644 index 00000000..76a97c0b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java @@ -0,0 +1,356 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyDescriptor; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.PropertyValues; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.support.ResourceEditorRegistrar; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.MessageSource; +import org.springframework.context.MessageSourceAware; +import org.springframework.core.convert.ConversionService; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.validation.*; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * Validate some {@link Properties} (or optionally + * {@link org.springframework.core.env.PropertySources}) by binding them to an + * object of a specified type and then optionally running a {@link Validator} + * over it. + * + * @param the target type + * @author Dave Syer + */ +public class PropertiesConfigurationFactory implements FactoryBean, ApplicationContextAware, MessageSourceAware, InitializingBean { + + private static final char[] EXACT_DELIMITERS = { '_', '.', '[' }; + + private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' }; + + private static final Log logger = LogFactory.getLog(PropertiesConfigurationFactory.class); + + private boolean ignoreUnknownFields = true; + + private boolean ignoreInvalidFields; + + private boolean exceptionIfInvalid = true; + + private PropertySources propertySources; + + private final T target; + + private Validator validator; + + private ApplicationContext applicationContext; + + private MessageSource messageSource; + + private boolean hasBeenBound = false; + + private boolean ignoreNestedProperties = false; + + private String targetName; + + private ConversionService conversionService; + + private boolean resolvePlaceholders = true; + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param target the target object to bind too + * @see #PropertiesConfigurationFactory(Class) + */ + public PropertiesConfigurationFactory(T target){ + Assert.notNull(target, "target must not be null"); + this.target = target; + } + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param type the target type + * @see #PropertiesConfigurationFactory(Class) + */ + @SuppressWarnings("unchecked") + public PropertiesConfigurationFactory(Class type){ + Assert.notNull(type, "type must not be null"); + this.target = (T) BeanUtils.instantiate(type); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set whether to ignore unknown fields, that is, whether to ignore bind + * parameters that do not have corresponding fields in the target object. + *

+ * Default is "true". Turn this off to enforce that all bind parameters must + * have a matching field in the target object. + * + * @param ignoreUnknownFields if unknown fields should be ignored + */ + public void setIgnoreUnknownFields(boolean ignoreUnknownFields) { + this.ignoreUnknownFields = ignoreUnknownFields; + } + + /** + * Set whether to ignore invalid fields, that is, whether to ignore bind + * parameters that have corresponding fields in the target object which are not + * accessible (for example because of null values in the nested path). + *

+ * Default is "false". Turn this on to ignore bind parameters for nested objects + * in non-existing parts of the target object graph. + * + * @param ignoreInvalidFields if invalid fields should be ignored + */ + public void setIgnoreInvalidFields(boolean ignoreInvalidFields) { + this.ignoreInvalidFields = ignoreInvalidFields; + } + + /** + * Set the target name. + * + * @param targetName the target name + */ + public void setTargetName(String targetName) { + this.targetName = targetName; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + } + + /** + * Set the message source. + * + * @param messageSource the message source + */ + @Override + public void setMessageSource(MessageSource messageSource) { + this.messageSource = messageSource; + } + + /** + * Set the property sources. + * + * @param propertySources the property sources + */ + public void setPropertySources(PropertySources propertySources) { + this.propertySources = propertySources; + } + + /** + * Set the conversion service. + * + * @param conversionService the conversion service + */ + public void setConversionService(ConversionService conversionService) { + this.conversionService = conversionService; + } + + /** + * Set the validator. + * + * @param validator the validator + */ + public void setValidator(Validator validator) { + this.validator = validator; + } + + /** + * Set a flag to indicate that an exception should be raised if a Validator is + * available and validation fails. + * + * @param exceptionIfInvalid the flag to set + * @deprecated as of 1.5, do not specify a {@link Validator} if validation + * should not occur + */ + @Deprecated + public void setExceptionIfInvalid(boolean exceptionIfInvalid) { + this.exceptionIfInvalid = exceptionIfInvalid; + } + + /** + * Flag to indicate that placeholders should be replaced during binding. Default + * is true. + * + * @param resolvePlaceholders flag value + */ + public void setResolvePlaceholders(boolean resolvePlaceholders) { + this.resolvePlaceholders = resolvePlaceholders; + } + + @Override + public void afterPropertiesSet() throws Exception { + bindPropertiesToTarget(); + } + + @Override + public Class getObjectType() { + if (this.target == null) { + return Object.class; + } + return this.target.getClass(); + } + + @Override + public boolean isSingleton() { + return true; + } + + @Override + public T getObject() throws Exception { + if (!this.hasBeenBound) { + bindPropertiesToTarget(); + } + return this.target; + } + + public void bindPropertiesToTarget() throws BindException { + Assert.state(this.propertySources != null, "PropertySources should not be null"); + try { + if (logger.isTraceEnabled()) { + logger.trace("Property Sources: " + this.propertySources); + + } + this.hasBeenBound = true; + doBindPropertiesToTarget(); + } catch (BindException ex) { + if (this.exceptionIfInvalid) { + throw ex; + } + logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex); + } + } + + private void doBindPropertiesToTarget() throws BindException { + RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target, + this.targetName) : new RelaxedDataBinder(this.target)); + if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) { + dataBinder.setValidator(this.validator); + } + if (this.conversionService != null) { + dataBinder.setConversionService(this.conversionService); + } + dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE); + dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties); + dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields); + dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields); + customizeBinder(dataBinder); + if (this.applicationContext != null) { + ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext, + this.applicationContext.getEnvironment()); + resourceEditorRegistrar.registerCustomEditors(dataBinder); + } + Iterable relaxedTargetNames = getRelaxedTargetNames(); + Set names = getNames(relaxedTargetNames); + PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames); + dataBinder.bind(propertyValues); + if (this.validator != null) { + dataBinder.validate(); + } + checkForBindingErrors(dataBinder); + } + + private Iterable getRelaxedTargetNames() { + return (this.target != null + && StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null); + } + + private Set getNames(Iterable prefixes) { + Set names = new LinkedHashSet(); + if (this.target != null) { + PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass()); + for (PropertyDescriptor descriptor : descriptors) { + String name = descriptor.getName(); + if (!name.equals("class")) { + RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name); + if (prefixes == null) { + for (String relaxedName : relaxedNames) { + names.add(relaxedName); + } + } else { + for (String prefix : prefixes) { + for (String relaxedName : relaxedNames) { + names.add(prefix + "." + relaxedName); + names.add(prefix + "_" + relaxedName); + } + } + } + } + } + } + return names; + } + + private PropertyValues getPropertySourcesPropertyValues(Set names, Iterable relaxedTargetNames) { + PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames); + return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders); + } + + private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set names, + Iterable relaxedTargetNames) { + if (this.ignoreUnknownFields && !isMapTarget()) { + // Since unknown fields are ignored we can filter them out early to save + // unnecessary calls to the PropertySource. + return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names); + } + if (relaxedTargetNames != null) { + // We can filter properties to those starting with the target name, but + // we can't do a complete filter since we need to trigger the + // unknown fields check + Set relaxedNames = new HashSet(); + for (String relaxedTargetName : relaxedTargetNames) { + relaxedNames.add(relaxedTargetName); + } + return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames); + } + // Not ideal, we basically can't filter anything + return PropertyNamePatternsMatcher.ALL; + } + + private boolean isMapTarget() { + return this.target != null && Map.class.isAssignableFrom(this.target.getClass()); + } + + private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException { + BindingResult errors = dataBinder.getBindingResult(); + if (errors.hasErrors()) { + logger.error("Properties configuration failed validation"); + for (ObjectError error : errors.getAllErrors()) { + logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault()) + + " (" + error + ")" : error); + } + if (this.exceptionIfInvalid) { + throw new BindException(errors); + } + } + } + + /** + * Customize the data binder. + * + * @param dataBinder the data binder that will be used to bind and validate + */ + protected void customizeBinder(DataBinder dataBinder) { + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java new file mode 100644 index 00000000..0d7d4fe4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +/** + * Strategy interface used to check if a property name matches specific + * criteria. + * + * @author Phillip Webb + * @since 1.2.0 + */ +interface PropertyNamePatternsMatcher { + + PropertyNamePatternsMatcher ALL = new PropertyNamePatternsMatcher() { + + @Override + public boolean matches(String propertyName) { + return true; + } + + }; + + PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() { + + @Override + public boolean matches(String propertyName) { + return false; + } + + }; + + /** + * Return {@code true} of the property name matches. + * + * @param propertyName the property name + * @return {@code true} if the property name matches + */ + boolean matches(String propertyName); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java new file mode 100644 index 00000000..7ad35f1a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java @@ -0,0 +1,30 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * The origin of a property, specifically its source and its name before any + * prefix was removed. + * + * @author Andy Wilkinson + * @since 1.3.0 + */ +public class PropertyOrigin { + + private final PropertySource source; + + private final String name; + + PropertyOrigin(PropertySource source, String name){ + this.name = name; + this.source = source; + } + + public PropertySource getSource() { + return this.source; + } + + public String getName() { + return this.name; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java new file mode 100644 index 00000000..520e857f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java @@ -0,0 +1,164 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.ConversionException; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.core.env.AbstractPropertyResolver; +import org.springframework.core.env.PropertyResolver; +import org.springframework.util.ClassUtils; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * {@link PropertyResolver} implementation that resolves property values against + * an underlying set of {@link PropertySources}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySource + * @see PropertySources + * @see AbstractEnvironment + * @since 3.1 + */ +public class PropertySourcesPropertyResolver extends AbstractPropertyResolver { + + private final PropertySources propertySources; + + /** + * Create a new resolver against the given property sources. + * + * @param propertySources the set of {@link PropertySource} objects to use + */ + public PropertySourcesPropertyResolver(PropertySources propertySources){ + this.propertySources = propertySources; + } + + @Override + public boolean containsProperty(String key) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(key)) { + return true; + } + } + } + return false; + } + + @Override + public String getProperty(String key) { + return getProperty(key, String.class, true); + } + + @Override + public T getProperty(String key, Class targetValueType) { + return getProperty(key, targetValueType, true); + } + + @Override + protected String getPropertyAsRawString(String key) { + return getProperty(key, String.class, false); + } + + protected T getProperty(String key, Class targetValueType, boolean resolveNestedPlaceholders) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger + .trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'"); + } + Object value = propertySource.getProperty(key); + if (value != null) { + if (resolveNestedPlaceholders && value instanceof String) { + value = resolveNestedPlaceholders((String) value); + } + logKeyFound(key, propertySource, value); + return convertValueIfNecessary(value, targetValueType); + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("Could not find key '" + key + "' in any property source"); + } + return null; + } + + @Deprecated + public Class getPropertyAsClass(String key, Class targetValueType) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName())); + } + Object value = propertySource.getProperty(key); + if (value != null) { + logKeyFound(key, propertySource, value); + Class clazz; + if (value instanceof String) { + try { + clazz = ClassUtils.forName((String) value, null); + } catch (Exception ex) { + throw new PropertySourcesPropertyResolver.ClassConversionException((String) value, + targetValueType, + ex); + } + } else if (value instanceof Class) { + clazz = (Class) value; + } else { + clazz = value.getClass(); + } + if (!targetValueType.isAssignableFrom(clazz)) { + throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType); + } + @SuppressWarnings("unchecked") + Class targetClass = (Class) clazz; + return targetClass; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug(String.format("Could not find key '%s' in any property source", key)); + } + return null; + } + + /** + * Log the given key as found in the given {@link PropertySource}, resulting in + * the given value. + *

+ * The default implementation writes a debug log message with key and source. As + * of 4.3.3, this does not log the value anymore in order to avoid accidental + * logging of sensitive settings. Subclasses may override this method to change + * the log level and/or log message, including the property's value if desired. + * + * @param key the key found + * @param propertySource the {@code PropertySource} that the key has been found + * in + * @param value the corresponding value + * @since 4.3.1 + */ + protected void logKeyFound(String key, PropertySource propertySource, Object value) { + if (logger.isDebugEnabled()) { + logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName() + + "' with value of type " + value.getClass().getSimpleName()); + } + } + + @SuppressWarnings("serial") + @Deprecated + private static class ClassConversionException extends ConversionException { + + public ClassConversionException(Class actual, Class expected){ + super(String + .format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName())); + } + + public ClassConversionException(String actual, Class expected, Exception ex){ + super( + String + .format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()), + ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java new file mode 100644 index 00000000..d41cb27b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java @@ -0,0 +1,233 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.util.Assert; +import org.springframework.validation.DataBinder; + +import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * A {@link PropertyValues} implementation backed by a {@link PropertySources}, + * bridging the two abstractions and allowing (for instance) a regular + * {@link DataBinder} to be used with the latter. + * + * @author Dave Syer + * @author Phillip Webb + */ +public class PropertySourcesPropertyValues implements PropertyValues { + + private static final Pattern COLLECTION_PROPERTY = Pattern + .compile("\\[(\\d+)\\](\\.\\S+)?"); + + private final PropertySources propertySources; + + private final Collection nonEnumerableFallbackNames; + + private final PropertyNamePatternsMatcher includes; + + private final Map propertyValues = new LinkedHashMap(); + + private final ConcurrentHashMap> collectionOwners = new ConcurrentHashMap>(); + + private final boolean resolvePlaceholders; + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + */ + public PropertySourcesPropertyValues(PropertySources propertySources){ + this(propertySources, true); + } + + /** + * Create a new PropertyValues from the given PropertySources that will + * optionally resolve placeholders. + * + * @param propertySources a PropertySources instance + * @param resolvePlaceholders {@code true} if placeholders should be resolved. + * @since 1.5.2 + */ + public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){ + this(propertySources, (Collection) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param includePatterns property name patterns to include from system + * properties and environment variables + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + */ + public PropertySourcesPropertyValues(PropertySources propertySources, Collection includePatterns, + Collection nonEnumerableFallbackNames){ + this(propertySources, + nonEnumerableFallbackNames, + new PatternPropertyNamePatternsMatcher(includePatterns), + true); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + * @param includes the property name patterns to include + * @param resolvePlaceholders flag to indicate the placeholders should be + * resolved + */ + PropertySourcesPropertyValues(PropertySources propertySources, Collection nonEnumerableFallbackNames, + PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){ + Assert.notNull(propertySources, "PropertySources must not be null"); + Assert.notNull(includes, "Includes must not be null"); + this.propertySources = propertySources; + this.nonEnumerableFallbackNames = nonEnumerableFallbackNames; + this.includes = includes; + this.resolvePlaceholders = resolvePlaceholders; + PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources); + for (PropertySource source : propertySources) { + processPropertySource(source, resolver); + } + } + + private void processPropertySource(PropertySource source, PropertySourcesPropertyResolver resolver) { + if (source instanceof CompositePropertySource) { + processCompositePropertySource((CompositePropertySource) source, resolver); + } else if (source instanceof EnumerablePropertySource) { + processEnumerablePropertySource((EnumerablePropertySource) source, resolver, this.includes); + } else { + processNonEnumerablePropertySource(source, resolver); + } + } + + private void processCompositePropertySource(CompositePropertySource source, + PropertySourcesPropertyResolver resolver) { + for (PropertySource nested : source.getPropertySources()) { + processPropertySource(nested, resolver); + } + } + + private void processEnumerablePropertySource(EnumerablePropertySource source, + PropertySourcesPropertyResolver resolver, + PropertyNamePatternsMatcher includes) { + if (source.getPropertyNames().length > 0) { + for (String propertyName : source.getPropertyNames()) { + if (includes.matches(propertyName)) { + Object value = getEnumerableProperty(source, resolver, propertyName); + putIfAbsent(propertyName, value, source); + } + } + } + } + + private Object getEnumerableProperty(EnumerablePropertySource source, PropertySourcesPropertyResolver resolver, + String propertyName) { + try { + if (this.resolvePlaceholders) { + return resolver.getProperty(propertyName, Object.class); + } + } catch (RuntimeException ex) { + // Probably could not resolve placeholders, ignore it here + } + return source.getProperty(propertyName); + } + + private void processNonEnumerablePropertySource(PropertySource source, + PropertySourcesPropertyResolver resolver) { + // We can only do exact matches for non-enumerable property names, but + // that's better than nothing... + if (this.nonEnumerableFallbackNames == null) { + return; + } + for (String propertyName : this.nonEnumerableFallbackNames) { + if (!source.containsProperty(propertyName)) { + continue; + } + Object value = null; + try { + value = resolver.getProperty(propertyName, Object.class); + } catch (RuntimeException ex) { + // Probably could not convert to Object, weird, but ignorable + } + if (value == null) { + value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH)); + } + putIfAbsent(propertyName, value, source); + } + } + + @Override + public PropertyValue[] getPropertyValues() { + Collection values = this.propertyValues.values(); + return values.toArray(new PropertyValue[values.size()]); + } + + @Override + public PropertyValue getPropertyValue(String propertyName) { + PropertyValue propertyValue = this.propertyValues.get(propertyName); + if (propertyValue != null) { + return propertyValue; + } + for (PropertySource source : this.propertySources) { + Object value = source.getProperty(propertyName); + propertyValue = putIfAbsent(propertyName, value, source); + if (propertyValue != null) { + return propertyValue; + } + } + return null; + } + + private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource source) { + if (value != null && !this.propertyValues.containsKey(propertyName)) { + PropertySource collectionOwner = this.collectionOwners + .putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source); + if (collectionOwner == null || collectionOwner == source) { + PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source); + this.propertyValues.put(propertyName, propertyValue); + return propertyValue; + } + } + return null; + } + + @Override + public PropertyValues changesSince(PropertyValues old) { + MutablePropertyValues changes = new MutablePropertyValues(); + // for each property value in the new set + for (PropertyValue newValue : getPropertyValues()) { + // if there wasn't an old one, add it + PropertyValue oldValue = old.getPropertyValue(newValue.getName()); + if (oldValue == null || !oldValue.equals(newValue)) { + changes.addPropertyValue(newValue); + } + } + return changes; + } + + @Override + public boolean contains(String propertyName) { + return getPropertyValue(propertyName) != null; + } + + @Override + public boolean isEmpty() { + return this.propertyValues.isEmpty(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java new file mode 100644 index 00000000..00a288b6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java @@ -0,0 +1,127 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; + +import org.springframework.core.convert.ConversionFailedException; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.convert.converter.ConverterFactory; +import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.util.Assert; + +/** + * Internal {@link ConversionService} used by {@link RelaxedDataBinder} to + * support additional relaxed conversion. + * + * @author Phillip Webb + * @author Stephane Nicoll + * @since 1.1.0 + */ +class RelaxedConversionService implements ConversionService { + + private final ConversionService conversionService; + + private final GenericConversionService additionalConverters; + + /** + * Create a new {@link RelaxedConversionService} instance. + * + * @param conversionService and option root conversion service + */ + RelaxedConversionService(ConversionService conversionService){ + this.conversionService = conversionService; + this.additionalConverters = new GenericConversionService(); + DefaultConversionService.addCollectionConverters(this.additionalConverters); + this.additionalConverters + .addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory()); + this.additionalConverters.addConverter(new StringToCharArrayConverter()); + } + + @Override + public boolean canConvert(Class sourceType, Class targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + @SuppressWarnings("unchecked") + public T convert(Object source, Class targetType) { + Assert.notNull(targetType, "The targetType to convert to cannot be null"); + return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType)); + } + + @Override + public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { + if (this.conversionService != null) { + try { + return this.conversionService.convert(source, sourceType, targetType); + } catch (ConversionFailedException ex) { + // Ignore and try the additional converters + } + } + return this.additionalConverters.convert(source, sourceType, targetType); + } + + /** + * Clone of Spring's package private StringToEnumConverterFactory, but ignoring + * the case of the source. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory { + + @Override + public Converter getConverter(Class targetType) { + Class enumType = targetType; + while (enumType != null && !enumType.isEnum()) { + enumType = enumType.getSuperclass(); + } + Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum"); + return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType); + } + + private class StringToEnum implements Converter { + + private final Class enumType; + + StringToEnum(Class enumType){ + this.enumType = enumType; + } + + @Override + public T convert(String source) { + if (source.isEmpty()) { + // It's an empty enum identifier: reset the enum value to null. + return null; + } + source = source.trim(); + for (T candidate : (Set) EnumSet.allOf(this.enumType)) { + RelaxedNames names = new RelaxedNames( + candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH)); + for (String name : names) { + if (name.equals(source)) { + return candidate; + } + } + if (candidate.name().equalsIgnoreCase(source)) { + return candidate; + } + } + throw new IllegalArgumentException( + "No enum constant " + this.enumType.getCanonicalName() + "." + source); + } + + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java new file mode 100644 index 00000000..dbcc7130 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java @@ -0,0 +1,729 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditor; +import java.net.InetAddress; +import java.util.*; + +import org.springframework.beans.*; +import org.springframework.beans.propertyeditors.FileEditor; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; +import org.springframework.validation.AbstractPropertyBindingResult; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.DataBinder; + +/** + * Binder implementation that allows caller to bind to maps and also allows + * property names to match a bit loosely (if underscores or dashes are removed + * and replaced with camel case for example). + * + * @author Dave Syer + * @author Phillip Webb + * @author Stephane Nicoll + * @author Andy Wilkinson + * @see RelaxedNames + */ +public class RelaxedDataBinder extends DataBinder { + + private static final Set> EXCLUDED_EDITORS; + + static { + Set> excluded = new HashSet>(); + excluded.add(FileEditor.class); + EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded); + } + + private static final Object BLANK = new Object(); + + private String namePrefix; + + private boolean ignoreNestedProperties; + + private MultiValueMap nameAliases = new LinkedMultiValueMap(); + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + */ + public RelaxedDataBinder(Object target){ + super(wrapTarget(target)); + } + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + * @param namePrefix An optional prefix to be used when reading properties + */ + public RelaxedDataBinder(Object target, String namePrefix){ + super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME)); + this.namePrefix = cleanNamePrefix(namePrefix); + } + + private String cleanNamePrefix(String namePrefix) { + if (!StringUtils.hasLength(namePrefix)) { + return null; + } + return (namePrefix.endsWith(".") ? namePrefix : namePrefix + "."); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set name aliases. + * + * @param aliases a map of property name to aliases + */ + public void setNameAliases(Map> aliases) { + this.nameAliases = new LinkedMultiValueMap(aliases); + } + + /** + * Add aliases to the {@link DataBinder}. + * + * @param name the property name to alias + * @param alias aliases for the property names + * @return this instance + */ + public RelaxedDataBinder withAlias(String name, String... alias) { + for (String value : alias) { + this.nameAliases.add(name, value); + } + return this; + } + + @Override + protected void doBind(MutablePropertyValues propertyValues) { + super.doBind(modifyProperties(propertyValues, getTarget())); + } + + /** + * Modify the property values so that period separated property paths are valid + * for map keys. Also creates new maps for properties of map type that are null + * (assuming all maps are potentially nested). The standard bracket {@code[...]} + * dereferencing is also accepted. + * + * @param propertyValues the property values + * @param target the target object + * @return modified property values + */ + private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) { + propertyValues = getPropertyValuesForNamePrefix(propertyValues); + if (target instanceof RelaxedDataBinder.MapHolder) { + propertyValues = addMapPrefix(propertyValues); + } + BeanWrapper wrapper = new BeanWrapperImpl(target); + wrapper.setConversionService(new RelaxedConversionService(getConversionService())); + wrapper.setAutoGrowNestedPaths(true); + List sortedValues = new ArrayList(); + Set modifiedNames = new HashSet(); + List sortedNames = getSortedPropertyNames(propertyValues); + for (String name : sortedNames) { + PropertyValue propertyValue = propertyValues.getPropertyValue(name); + PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue); + if (modifiedNames.add(modifiedProperty.getName())) { + sortedValues.add(modifiedProperty); + } + } + return new MutablePropertyValues(sortedValues); + } + + private List getSortedPropertyNames(MutablePropertyValues propertyValues) { + List names = new LinkedList(); + for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) { + names.add(propertyValue.getName()); + } + sortPropertyNames(names); + return names; + } + + /** + * Sort by name so that parent properties get processed first (e.g. 'foo.bar' + * before 'foo.bar.spam'). Don't use Collections.sort() because the order might + * be significant for other property names (it shouldn't be but who knows what + * people might be relying on, e.g. HSQL has a JDBCXADataSource where + * "databaseName" is a synonym for "url"). + * + * @param names the names to sort + */ + private void sortPropertyNames(List names) { + for (String name : new ArrayList(names)) { + int propertyIndex = names.indexOf(name); + RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name); + for (String prefix : path.prefixes()) { + int prefixIndex = names.indexOf(prefix); + if (prefixIndex >= propertyIndex) { + // The child property has a parent in the list in the wrong order + names.remove(name); + names.add(prefixIndex, name); + } + } + } + } + + private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) { + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue pv : propertyValues.getPropertyValues()) { + rtn.add("map." + pv.getName(), pv.getValue()); + } + return rtn; + } + + private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) { + if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) { + return propertyValues; + } + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue value : propertyValues.getPropertyValues()) { + String name = value.getName(); + for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) { + for (String separator : new String[] { ".", "_" }) { + String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix); + if (name.startsWith(candidate)) { + name = name.substring(candidate.length()); + if (!(this.ignoreNestedProperties && name.contains("."))) { + PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value); + rtn.addPropertyValue( + new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin)); + } + } + } + } + } + return rtn; + } + + private String stripLastDot(String string) { + if (StringUtils.hasLength(string) && string.endsWith(".")) { + string = string.substring(0, string.length() - 1); + } + return string; + } + + private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) { + String name = propertyValue.getName(); + String normalizedName = normalizePath(target, name); + if (!normalizedName.equals(name)) { + return new PropertyValue(normalizedName, propertyValue.getValue()); + } + return propertyValue; + } + + /** + * Normalize a bean property path to a format understood by a BeanWrapper. This + * is used so that + *

    + *
  • Fuzzy matching can be employed for bean property names
    • + *
  • Period separators can be used instead of indexing ([...]) for map + * keys
    • + *
+ * + * @param wrapper a bean wrapper for the object to bind + * @param path the bean path to bind + * @return a transformed path with correct bean wrapper syntax + */ + protected String normalizePath(BeanWrapper wrapper, String path) { + return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0); + } + + @Override + protected AbstractPropertyBindingResult createBeanPropertyBindingResult() { + return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(), + getObjectName(), + isAutoGrowNestedPaths(), + getAutoGrowCollectionLimit(), + getConversionService()); + } + + private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String prefix = path.prefix(index); + String key = path.name(index); + if (path.isProperty(index)) { + key = getActualPropertyName(wrapper, prefix, key); + path.rename(index, key); + } + if (path.name(++index) == null) { + return path.toString(); + } + String name = path.prefix(index); + TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name); + if (descriptor == null || descriptor.isMap()) { + if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + extendMapIfNecessary(wrapper, path, index); + } else if (descriptor.isCollection()) { + extendCollectionIfNecessary(wrapper, path, index); + } else if (descriptor.getType().equals(Object.class)) { + if (isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + if (path.isLastNode(index)) { + wrapper.setPropertyValue(path.toString(), BLANK); + } else { + String next = path.prefix(index + 1); + if (wrapper.getPropertyValue(next) == null) { + wrapper.setPropertyValue(next, new LinkedHashMap()); + } + } + } + return initializePath(wrapper, path, index); + } + + private boolean isMapValueStringType(TypeDescriptor descriptor) { + if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) { + return false; + } + if (Properties.class.isAssignableFrom(descriptor.getObjectType())) { + // Properties is declared as Map but we know it's really + // Map + return true; + } + Class valueType = descriptor.getMapValueTypeDescriptor().getObjectType(); + return (valueType != null && CharSequence.class.isAssignableFrom(valueType)); + } + + @SuppressWarnings("rawtypes") + private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) { + Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null); + if (value instanceof Map) { + if (((Map) value).get(key) == BLANK) { + return true; + } + } + return false; + } + + private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor(); + if (!elementDescriptor.isMap() && !elementDescriptor.isCollection() + && !elementDescriptor.getType().equals(Object.class)) { + return; + } + Object extend = new LinkedHashMap(); + if (!elementDescriptor.isMap() && path.isArrayIndex(index)) { + extend = new ArrayList(); + } + wrapper.setPropertyValue(path.prefix(index + 1), extend); + } + + private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name); + if (parent == null) { + return; + } + TypeDescriptor descriptor = parent.getMapValueTypeDescriptor(); + if (descriptor == null) { + descriptor = TypeDescriptor.valueOf(Object.class); + } + if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) { + return; + } + String extensionName = path.prefix(index + 1); + if (wrapper.isReadableProperty(extensionName)) { + Object currentValue = wrapper.getPropertyValue(extensionName); + if ((descriptor.isCollection() && currentValue instanceof Collection) + || (!descriptor.isCollection() && currentValue instanceof Map)) { + return; + } + } + Object extend = new LinkedHashMap(); + if (descriptor.isCollection()) { + extend = new ArrayList(); + } + if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) { + extend = BLANK; + } + wrapper.setPropertyValue(extensionName, extend); + } + + private String getActualPropertyName(BeanWrapper target, String prefix, String name) { + String propertyName = resolvePropertyName(target, prefix, name); + if (propertyName == null) { + propertyName = resolveNestedPropertyName(target, prefix, name); + } + return (propertyName != null ? propertyName : name); + } + + private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) { + StringBuilder candidate = new StringBuilder(); + for (String field : name.split("[_\\-\\.]")) { + candidate.append(candidate.length() > 0 ? "." : ""); + candidate.append(field); + String nested = resolvePropertyName(target, prefix, candidate.toString()); + if (nested != null) { + Class type = target.getPropertyType(nested); + if ((type != null) && Map.class.isAssignableFrom(type)) { + // Special case for map property (gh-3836). + return nested + "[" + name.substring(candidate.length() + 1) + "]"; + } + String propertyName = resolvePropertyName(target, + joinString(prefix, nested), + name.substring(candidate.length() + 1)); + if (propertyName != null) { + return joinString(nested, propertyName); + } + } + } + return null; + } + + private String resolvePropertyName(BeanWrapper target, String prefix, String name) { + Iterable names = getNameAndAliases(name); + for (String nameOrAlias : names) { + for (String candidate : new RelaxedNames(nameOrAlias)) { + try { + if (target.getPropertyType(joinString(prefix, candidate)) != null) { + return candidate; + } + } catch (InvalidPropertyException ex) { + // swallow and continue + } + } + } + return null; + } + + private String joinString(String prefix, String name) { + return (StringUtils.hasLength(prefix) ? prefix + "." + name : name); + } + + private Iterable getNameAndAliases(String name) { + List aliases = this.nameAliases.get(name); + if (aliases == null) { + return Collections.singleton(name); + } + List nameAndAliases = new ArrayList(aliases.size() + 1); + nameAndAliases.add(name); + nameAndAliases.addAll(aliases); + return nameAndAliases; + } + + private static Object wrapTarget(Object target) { + if (target instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) target; + target = new RelaxedDataBinder.MapHolder(map); + } + return target; + } + + @Override + public void registerCustomEditor(Class requiredType, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, propertyEditor); + } + } + + @Override + public void registerCustomEditor(Class requiredType, String field, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, field, propertyEditor); + } + } + + /** + * Holder to allow Map targets to be bound. + */ + static class MapHolder { + + private Map map; + + MapHolder(Map map){ + this.map = map; + } + + public void setMap(Map map) { + this.map = map; + } + + public Map getMap() { + return this.map; + } + + } + + /** + * A path though properties of a bean. + */ + private static class BeanPath { + + private List nodes; + + BeanPath(String path){ + this.nodes = splitPath(path); + } + + public List prefixes() { + List prefixes = new ArrayList(); + for (int index = 1; index < this.nodes.size(); index++) { + prefixes.add(prefix(index)); + } + return prefixes; + } + + public boolean isLastNode(int index) { + return index >= this.nodes.size() - 1; + } + + private List splitPath(String path) { + List nodes = new ArrayList(); + String current = extractIndexedPaths(path, nodes); + for (String name : StringUtils.delimitedListToStringArray(current, ".")) { + if (StringUtils.hasText(name)) { + nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name)); + } + } + return nodes; + } + + private String extractIndexedPaths(String path, List nodes) { + int startRef = path.indexOf("["); + String current = path; + while (startRef >= 0) { + if (startRef > 0) { + nodes.addAll(splitPath(current.substring(0, startRef))); + } + int endRef = current.indexOf("]", startRef); + if (endRef > 0) { + String sub = current.substring(startRef + 1, endRef); + if (sub.matches("[0-9]+")) { + nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub)); + } else { + nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub)); + } + } + current = current.substring(endRef + 1); + startRef = current.indexOf("["); + } + return current; + } + + public void collapseKeys(int index) { + List revised = new ArrayList(); + for (int i = 0; i < index; i++) { + revised.add(this.nodes.get(i)); + } + StringBuilder builder = new StringBuilder(); + for (int i = index; i < this.nodes.size(); i++) { + if (i > index) { + builder.append("."); + } + builder.append(this.nodes.get(i).name); + } + revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString())); + this.nodes = revised; + } + + public void mapIndex(int index) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index); + if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) { + node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex(); + } + this.nodes.set(index, node); + } + + public String prefix(int index) { + return range(0, index); + } + + public void rename(int index, String name) { + this.nodes.get(index).name = name; + } + + public String name(int index) { + if (index < this.nodes.size()) { + return this.nodes.get(index).name; + } + return null; + } + + private String range(int start, int end) { + StringBuilder builder = new StringBuilder(); + for (int i = start; i < end; i++) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i); + builder.append(node); + } + if (builder.toString().startsWith(("."))) { + builder.replace(0, 1, ""); + } + return builder.toString(); + } + + public boolean isArrayIndex(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode; + } + + public boolean isProperty(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode; + } + + @Override + public String toString() { + return prefix(this.nodes.size()); + } + + private static class PathNode { + + protected String name; + + PathNode(String name){ + this.name = name; + } + + } + + private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + ArrayIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + MapIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode { + + PropertyNode(String name){ + super(name); + } + + public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() { + return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name); + } + + @Override + public String toString() { + return "." + this.name; + } + + } + + } + + /** + * Extended version of {@link BeanPropertyBindingResult} to support relaxed + * binding. + */ + private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult { + + private RelaxedConversionService conversionService; + + RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths, + int autoGrowCollectionLimit, ConversionService conversionService){ + super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit); + this.conversionService = new RelaxedConversionService(conversionService); + } + + @Override + protected BeanWrapper createBeanWrapper() { + BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget()); + beanWrapper.setConversionService(this.conversionService); + beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor()); + return beanWrapper; + } + + } + + /** + * Extended version of {@link BeanWrapperImpl} to support relaxed binding. + */ + private static class RelaxedBeanWrapper extends BeanWrapperImpl { + + private static final Set BENIGN_PROPERTY_SOURCE_NAMES; + + static { + Set names = new HashSet(); + names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); + names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); + BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names); + } + + RelaxedBeanWrapper(Object target){ + super(target); + } + + @Override + public void setPropertyValue(PropertyValue pv) throws BeansException { + try { + super.setPropertyValue(pv); + } catch (NotWritablePropertyException ex) { + PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv); + if (isBenign(origin)) { + logger.debug("Ignoring benign property binding failure", ex); + return; + } + if (origin == null) { + throw ex; + } + throw new RelaxedBindingNotWritablePropertyException(ex, origin); + } + } + + private boolean isBenign(PropertyOrigin origin) { + String name = (origin != null ? origin.getSource().getName() : null); + return BENIGN_PROPERTY_SOURCE_NAMES.contains(name); + } + + } + + public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException { + + private final String message; + + private final PropertyOrigin propertyOrigin; + + RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){ + super(ex.getBeanClass(), ex.getPropertyName()); + this.propertyOrigin = propertyOrigin; + this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '" + + propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '" + + ex.getBeanClass().getName() + "'"; + } + + @Override + public String getMessage() { + return this.message; + } + + public PropertyOrigin getPropertyOrigin() { + return this.propertyOrigin; + } + + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java new file mode 100644 index 00000000..f1b1628f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java @@ -0,0 +1,241 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * Generates relaxed name variations from a given source. + * + * @author Phillip Webb + * @author Dave Syer + * @see RelaxedDataBinder + */ +public final class RelaxedNames implements Iterable { + + private static final Pattern CAMEL_CASE_PATTERN = Pattern.compile("([^A-Z-])([A-Z])"); + + private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]"); + + private final String name; + + private final Set values = new LinkedHashSet(); + + /** + * Create a new {@link RelaxedNames} instance. + * + * @param name the source name. For the maximum number of variations specify the + * name using dashed notation (e.g. {@literal my-property-name} + */ + public RelaxedNames(String name){ + this.name = (name != null ? name : ""); + initialize(RelaxedNames.this.name, this.values); + } + + @Override + public Iterator iterator() { + return this.values.iterator(); + } + + private void initialize(String name, Set values) { + if (values.contains(name)) { + return; + } + for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) { + for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) { + String result = name; + result = manipulation.apply(result); + result = variation.apply(result); + values.add(result); + initialize(result, values); + } + } + } + + /** + * Name variations. + */ + enum Variation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + LOWERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH)); + } + + }, + + UPPERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH)); + } + + }; + + public abstract String apply(String value); + + } + + /** + * Name manipulations. + */ + enum Manipulation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + HYPHEN_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('-') != -1 ? value.replace('-', '_') : value); + } + + }, + + UNDERSCORE_TO_PERIOD { + + @Override + public String apply(String value) { + return (value.indexOf('_') != -1 ? value.replace('_', '.') : value); + } + + }, + + PERIOD_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('.') != -1 ? value.replace('.', '_') : value); + } + + }, + + CAMELCASE_TO_UNDERSCORE { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + CAMELCASE_TO_HYPHEN { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, false); + } + + }, + + CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, true); + } + + }; + + private static final char[] SUFFIXES = new char[] { '_', '-', '.' }; + + public abstract String apply(String value); + + private static String separatedToCamelCase(String value, boolean caseInsensitive) { + if (value.isEmpty()) { + return value; + } + StringBuilder builder = new StringBuilder(); + for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) { + field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field); + builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field); + } + char lastChar = value.charAt(value.length() - 1); + for (char suffix : SUFFIXES) { + if (lastChar == suffix) { + builder.append(suffix); + break; + } + } + return builder.toString(); + } + + } + + /** + * Return a {@link RelaxedNames} for the given source camelCase source name. + * + * @param name the source name in camelCase + * @return the relaxed names + */ + public static RelaxedNames forCamelCase(String name) { + StringBuilder result = new StringBuilder(); + for (char c : name.toCharArray()) { + result.append(Character.isUpperCase(c) && result.length() > 0 + && result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c); + } + return new RelaxedNames(result.toString()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java new file mode 100644 index 00000000..00a2d808 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java @@ -0,0 +1,17 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.converter.Converter; + +/** + * Converts a String to a Char Array. + * + * @author Phillip Webb + */ +class StringToCharArrayConverter implements Converter { + + @Override + public char[] convert(String source) { + return source.toCharArray(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java new file mode 100644 index 00000000..9cb73e90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java @@ -0,0 +1,203 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import org.springframework.util.Assert; +import org.springframework.util.ResourceUtils; + +/** + * Convenience base class for {@link Resource} implementations, pre-implementing + * typical behavior. + *

+ * The "exists" method will check whether a File or InputStream can be opened; + * "isOpen" will always return false; "getURL" and "getFile" throw an exception; + * and "toString" will return the description. + * + * @author Juergen Hoeller + * @since 28.12.2003 + */ +public abstract class AbstractResource implements Resource { + + /** + * This implementation checks whether a File can be opened, falling back to + * whether an InputStream can be opened. This will cover both directories and + * content resources. + */ + @Override + public boolean exists() { + // Try file existence: can we find the file in the file system? + try { + return getFile().exists(); + } catch (IOException ex) { + // Fall back to stream existence: can we open the stream? + try { + InputStream is = getInputStream(); + is.close(); + return true; + } catch (Throwable isEx) { + return false; + } + } + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean isReadable() { + return true; + } + + /** + * This implementation always returns {@code false}. + */ + @Override + public boolean isOpen() { + return false; + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to a URL. + */ + @Override + public URL getURL() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to URL"); + } + + /** + * This implementation builds a URI based on the URL returned by + * {@link #getURL()}. + */ + @Override + public URI getURI() throws IOException { + URL url = getURL(); + try { + return ResourceUtils.toURI(url); + } catch (URISyntaxException ex) { + throw new RuntimeException("Invalid URI [" + url + "]", ex); + } + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to an absolute file path. + */ + @Override + public File getFile() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path"); + } + + /** + * This implementation reads the entire InputStream to calculate the content + * length. Subclasses will almost always be able to provide a more optimal + * version of this, e.g. checking a File length. + * + * @see #getInputStream() + */ + @Override + public long contentLength() throws IOException { + InputStream is = getInputStream(); + Assert.state(is != null, "Resource InputStream must not be null"); + try { + long size = 0; + byte[] buf = new byte[255]; + int read; + while ((read = is.read(buf)) != -1) { + size += read; + } + return size; + } finally { + try { + is.close(); + } catch (IOException ex) { + } + } + } + + /** + * This implementation checks the timestamp of the underlying File, if + * available. + * + * @see #getFileForLastModifiedCheck() + */ + @Override + public long lastModified() throws IOException { + long lastModified = getFileForLastModifiedCheck().lastModified(); + if (lastModified == 0L) { + throw new FileNotFoundException( + getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp"); + } + return lastModified; + } + + /** + * Determine the File to use for timestamp checking. + *

+ * The default implementation delegates to {@link #getFile()}. + * + * @return the File to use for timestamp checking (never {@code null}) + * @throws FileNotFoundException if the resource cannot be resolved as an + * absolute file path, i.e. is not available in a file system + * @throws IOException in case of general resolution/reading failures + */ + protected File getFileForLastModifiedCheck() throws IOException { + return getFile(); + } + + /** + * This implementation throws a FileNotFoundException, assuming that relative + * resources cannot be created for this resource. + */ + @Override + public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException { + throw new FileNotFoundException("Cannot create a relative resource for " + getDescription()); + } + + /** + * This implementation always returns {@code null}, assuming that this resource + * type does not have a filename. + */ + @Override + public String getFilename() { + return null; + } + + /** + * This implementation returns the description of this resource. + * + * @see #getDescription() + */ + @Override + public String toString() { + return getDescription(); + } + + /** + * This implementation compares description strings. + * + * @see #getDescription() + */ + @Override + public boolean equals(Object obj) { + return (obj == this + || (obj instanceof org.springframework.core.io.Resource + && ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription()))); + } + + /** + * This implementation returns the description's hash code. + * + * @see #getDescription() + */ + @Override + public int hashCode() { + return getDescription().hashCode(); + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java new file mode 100644 index 00000000..3ee85766 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java @@ -0,0 +1,118 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.InputStreamSource; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * {@link Resource} implementation for a given byte array. + *

+ * Creates a {@link ByteArrayInputStream} for the given byte array. + *

+ * Useful for loading content from any given byte array, without having to + * resort to a single-use {@link InputStreamResource}. Particularly useful for + * creating mail attachments from local content, where JavaMail needs to be able + * to read the stream multiple times. + * + * @author Juergen Hoeller + * @author Sam Brannen + * @see ByteArrayInputStream + * @see InputStreamResource + * @since 1.2.3 + */ +public class ByteArrayResource extends AbstractResource { + + private final byte[] byteArray; + + private final String description; + + /** + * Create a new {@code ByteArrayResource}. + * + * @param byteArray the byte array to wrap + */ + public ByteArrayResource(byte[] byteArray){ + this(byteArray, "resource loaded from byte array"); + } + + /** + * Create a new {@code ByteArrayResource} with a description. + * + * @param byteArray the byte array to wrap + * @param description where the byte array comes from + */ + public ByteArrayResource(byte[] byteArray, String description){ + Assert.notNull(byteArray, "Byte array must not be null"); + this.byteArray = byteArray; + this.description = (description != null ? description : ""); + } + + /** + * Return the underlying byte array. + */ + public final byte[] getByteArray() { + return this.byteArray; + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean exists() { + return true; + } + + /** + * This implementation returns the length of the underlying byte array. + */ + @Override + public long contentLength() { + return this.byteArray.length; + } + + /** + * This implementation returns a ByteArrayInputStream for the underlying byte + * array. + * + * @see ByteArrayInputStream + */ + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.byteArray); + } + + /** + * This implementation returns a description that includes the passed-in + * {@code description}, if any. + */ + @Override + public String getDescription() { + return "Byte array resource [" + this.description + "]"; + } + + /** + * This implementation compares the underlying byte array. + * + * @see Arrays#equals(byte[], byte[]) + */ + @Override + public boolean equals(Object obj) { + return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource + && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); + } + + /** + * This implementation returns the hash code based on the underlying byte array. + */ + @Override + public int hashCode() { + return (byte[].class.hashCode() * 29 * this.byteArray.length); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java new file mode 100644 index 00000000..106f8f90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java @@ -0,0 +1,107 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.util.StringUtils; + +/** + * Composite {@link PropertySource} implementation that iterates over a set of + * {@link PropertySource} instances. Necessary in cases where multiple property + * sources share the same name, e.g. when multiple values are supplied to + * {@code @PropertySource}. + *

+ * As of Spring 4.1.2, this class extends {@link EnumerablePropertySource} + * instead of plain {@link PropertySource}, exposing {@link #getPropertyNames()} + * based on the accumulated property names from all contained sources (as far as + * possible). + * + * @author Chris Beams + * @author Juergen Hoeller + * @author Phillip Webb + * @since 3.1.1 + */ +public class CompositePropertySource extends EnumerablePropertySource { + + private final Set> propertySources = new LinkedHashSet>(); + + /** + * Create a new {@code CompositePropertySource}. + * + * @param name the name of the property source + */ + public CompositePropertySource(String name){ + super(name); + } + + @Override + public Object getProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + Object candidate = propertySource.getProperty(name); + if (candidate != null) { + return candidate; + } + } + return null; + } + + @Override + public boolean containsProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(name)) { + return true; + } + } + return false; + } + + @Override + public String[] getPropertyNames() { + Set names = new LinkedHashSet(); + for (PropertySource propertySource : this.propertySources) { + if (!(propertySource instanceof EnumerablePropertySource)) { + throw new IllegalStateException( + "Failed to enumerate property names due to non-enumerable property source: " + propertySource); + } + names.addAll(Arrays.asList(((EnumerablePropertySource) propertySource).getPropertyNames())); + } + return StringUtils.toStringArray(names); + } + + /** + * Add the given {@link PropertySource} to the end of the chain. + * + * @param propertySource the PropertySource to add + */ + public void addPropertySource(PropertySource propertySource) { + this.propertySources.add(propertySource); + } + + /** + * Add the given {@link PropertySource} to the start of the chain. + * + * @param propertySource the PropertySource to add + * @since 4.1 + */ + public void addFirstPropertySource(PropertySource propertySource) { + List> existing = new ArrayList>(this.propertySources); + this.propertySources.clear(); + this.propertySources.add(propertySource); + this.propertySources.addAll(existing); + } + + /** + * Return all property sources that this composite source holds. + * + * @since 4.1.1 + */ + public Collection> getPropertySources() { + return this.propertySources; + } + + @Override + public String toString() { + return String + .format("%s [name='%s', propertySources=%s]", getClass().getSimpleName(), this.name, this.propertySources); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java new file mode 100644 index 00000000..e5ce8639 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.springframework.util.ObjectUtils; + +/** + * A {@link PropertySource} implementation capable of interrogating its + * underlying source object to enumerate all possible property name/value pairs. + * Exposes the {@link #getPropertyNames()} method to allow callers to introspect + * available properties without having to access the underlying source object. + * This also facilitates a more efficient implementation of + * {@link #containsProperty(String)}, in that it can call + * {@link #getPropertyNames()} and iterate through the returned array rather + * than attempting a call to {@link #getProperty(String)} which may be more + * expensive. Implementations may consider caching the result of + * {@link #getPropertyNames()} to fully exploit this performance opportunity. + *

+ * Most framework-provided {@code PropertySource} implementations are + * enumerable; a counter-example would be {@code JndiPropertySource} where, due + * to the nature of JNDI it is not possible to determine all possible property + * names at any given time; rather it is only possible to try to access a + * property (via {@link #getProperty(String)}) in order to evaluate whether it + * is present or not. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public abstract class EnumerablePropertySource extends PropertySource { + + public EnumerablePropertySource(String name, T source){ + super(name, source); + } + + protected EnumerablePropertySource(String name){ + super(name); + } + + /** + * Return whether this {@code PropertySource} contains a property with the given + * name. + *

+ * This implementation checks for the presence of the given name within the + * {@link #getPropertyNames()} array. + * + * @param name the name of the property to find + */ + @Override + public boolean containsProperty(String name) { + return ObjectUtils.containsElement(getPropertyNames(), name); + } + + /** + * Return the names of all properties contained by the {@linkplain #getSource() + * source} object (never {@code null}). + */ + public abstract String[] getPropertyNames(); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java new file mode 100644 index 00000000..eecc0ee4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; + +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; +import org.springframework.util.StringUtils; + +/** + * {@link PropertySource} that reads keys and values from a {@code Map} object. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + * @see PropertiesPropertySource + */ +public class MapPropertySource extends EnumerablePropertySource> { + + public MapPropertySource(String name, Map source){ + super(name, source); + } + + @Override + public Object getProperty(String name) { + return this.source.get(name); + } + + @Override + public boolean containsProperty(String name) { + return this.source.containsKey(name); + } + + @Override + public String[] getPropertyNames() { + return StringUtils.toStringArray(this.source.keySet()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java new file mode 100644 index 00000000..d8f32d3c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java @@ -0,0 +1,221 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.core.env.PropertyResolver; +import org.springframework.core.env.PropertySourcesPropertyResolver; + +/** + * Default implementation of the {@link PropertySources} interface. Allows + * manipulation of contained property sources and provides a constructor for + * copying an existing {@code PropertySources} instance. + *

+ * Where precedence is mentioned in methods such as {@link #addFirst} + * and {@link #addLast}, this is with regard to the order in which property + * sources will be searched when resolving a given property with a + * {@link PropertyResolver}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySourcesPropertyResolver + * @since 3.1 + */ +public class MutablePropertySources implements PropertySources { + + private final Log logger; + + private final List> propertySourceList = new CopyOnWriteArrayList>(); + + /** + * Create a new {@link MutablePropertySources} + * object. + */ + public MutablePropertySources(){ + this.logger = LogFactory.getLog(getClass()); + } + + /** + * Create a new {@code MutablePropertySources} from the given propertySources + * object, preserving the original order of contained {@code PropertySource} + * objects. + */ + public MutablePropertySources(PropertySources propertySources){ + this(); + for (PropertySource propertySource : propertySources) { + addLast(propertySource); + } + } + + /** + * Create a new {@link MutablePropertySources} + * object and inherit the given logger, usually from an enclosing + * {@link Environment}. + */ + MutablePropertySources(Log logger){ + this.logger = logger; + } + + @Override + public boolean contains(String name) { + return this.propertySourceList.contains(PropertySource.named(name)); + } + + @Override + public PropertySource get(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.get(index) : null); + } + + @Override + public Iterator> iterator() { + return this.propertySourceList.iterator(); + } + + /** + * Add the given property source object with highest precedence. + */ + public void addFirst(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with highest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(0, propertySource); + } + + /** + * Add the given property source object with lowest precedence. + */ + public void addLast(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with lowest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(propertySource); + } + + /** + * Add the given property source object with precedence immediately higher than + * the named relative property source. + */ + public void addBefore(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately higher than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index, propertySource); + } + + /** + * Add the given property source object with precedence immediately lower than + * the named relative property source. + */ + public void addAfter(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately lower than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index + 1, propertySource); + } + + /** + * Return the precedence of the given property source, {@code -1} if not found. + */ + public int precedenceOf(PropertySource propertySource) { + return this.propertySourceList.indexOf(propertySource); + } + + /** + * Remove and return the property source with the given name, {@code null} if + * not found. + * + * @param name the name of the property source to find and remove + */ + public PropertySource remove(String name) { + if (logger.isDebugEnabled()) { + logger.debug("Removing PropertySource '" + name + "'"); + } + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.remove(index) : null); + } + + /** + * Replace the property source with the given name with the given property + * source object. + * + * @param name the name of the property source to find and replace + * @param propertySource the replacement property source + * @throws IllegalArgumentException if no property source with the given name is + * present + * @see #contains + */ + public void replace(String name, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Replacing PropertySource '" + name + "' with '" + propertySource.getName() + "'"); + } + int index = assertPresentAndGetIndex(name); + this.propertySourceList.set(index, propertySource); + } + + /** + * Return the number of {@link PropertySource} objects contained. + */ + public int size() { + return this.propertySourceList.size(); + } + + @Override + public String toString() { + return this.propertySourceList.toString(); + } + + /** + * Ensure that the given property source is not being added relative to itself. + */ + protected void assertLegalRelativeAddition(String relativePropertySourceName, PropertySource propertySource) { + String newPropertySourceName = propertySource.getName(); + if (relativePropertySourceName.equals(newPropertySourceName)) { + throw new IllegalArgumentException( + "PropertySource named '" + newPropertySourceName + "' cannot be added relative to itself"); + } + } + + /** + * Remove the given property source if it is present. + */ + protected void removeIfPresent(PropertySource propertySource) { + this.propertySourceList.remove(propertySource); + } + + /** + * Add the given property source at a particular index in the list. + */ + private void addAtIndex(int index, PropertySource propertySource) { + removeIfPresent(propertySource); + this.propertySourceList.add(index, propertySource); + } + + /** + * Assert that the named property source is present and return its index. + * + * @param name {@linkplain PropertySource#getName() name of the property source} + * to find + * @throws IllegalArgumentException if the named property source is not present + */ + private int assertPresentAndGetIndex(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + if (index == -1) { + throw new IllegalArgumentException("PropertySource named '" + name + "' does not exist"); + } + return index; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java new file mode 100644 index 00000000..12fe686a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; +import java.util.Properties; + +import org.springframework.core.env.PropertySource; + +/** + * {@link PropertySource} implementation that extracts properties from a + * {@link Properties} object. + *

+ * Note that because a {@code Properties} object is technically an + * {@code } {@link java.util.Hashtable Hashtable}, one may + * contain non-{@code String} keys or values. This implementation, however is + * restricted to accessing only {@code String}-based keys and values, in the + * same fashion as {@link Properties#getProperty} and + * {@link Properties#setProperty}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public class PropertiesPropertySource extends MapPropertySource { + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public PropertiesPropertySource(String name, Properties source){ + super(name, (Map) source); + } + + protected PropertiesPropertySource(String name, Map source){ + super(name, source); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java new file mode 100644 index 00000000..f7e4cc34 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java @@ -0,0 +1,239 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * Abstract base class representing a source of name/value property pairs. The + * underlying {@linkplain #getSource() source object} may be of any type + * {@code T} that encapsulates properties. Examples include + * {@link java.util.Properties} objects, {@link java.util.Map} objects, + * {@code ServletContext} and {@code ServletConfig} objects (for access to init + * parameters). Explore the {@code PropertySource} type hierarchy to see + * provided implementations. + *

+ * {@code PropertySource} objects are not typically used in isolation, but + * rather through a {@link PropertySources} object, which aggregates property + * sources and in conjunction with a {@link PropertyResolver} implementation + * that can perform precedence-based searches across the set of + * {@code PropertySources}. + *

+ * {@code PropertySource} identity is determined not based on the content of + * encapsulated properties, but rather based on the {@link #getName() name} of + * the {@code PropertySource} alone. This is useful for manipulating + * {@code PropertySource} objects when in collection contexts. See operations in + * {@link MutablePropertySources} as well as the {@link #named(String)} and + * {@link #toString()} methods for details. + *

+ * Note that when working + * with @{@link org.springframework.context.annotation.Configuration + * Configuration} classes that the @{@link PropertySource PropertySource} + * annotation provides a convenient and declarative way of adding property + * sources to the enclosing {@code Environment}. + * + * @author Chris Beams + * @since 3.1 + * @see PropertySources + * @see MutablePropertySources + * @see PropertySource + */ +public abstract class PropertySource { + + protected final Log logger = LogFactory.getLog(getClass()); + + protected final String name; + + protected final T source; + + /** + * Create a new {@code PropertySource} with the given name and source object. + */ + public PropertySource(String name, T source){ + Assert.hasText(name, "Property source name must contain at least one character"); + Assert.notNull(source, "Property source must not be null"); + this.name = name; + this.source = source; + } + + /** + * Create a new {@code PropertySource} with the given name and with a new + * {@code Object} instance as the underlying source. + *

+ * Often useful in testing scenarios when creating anonymous implementations + * that never query an actual source but rather return hard-coded values. + */ + @SuppressWarnings("unchecked") + public PropertySource(String name){ + this(name, (T) new Object()); + } + + /** + * Return the name of this {@code PropertySource} + */ + public String getName() { + return this.name; + } + + /** + * Return the underlying source object for this {@code PropertySource}. + */ + public T getSource() { + return this.source; + } + + /** + * Return whether this {@code PropertySource} contains the given name. + *

+ * This implementation simply checks for a {@code null} return value from + * {@link #getProperty(String)}. Subclasses may wish to implement a more + * efficient algorithm if possible. + * + * @param name the property name to find + */ + public boolean containsProperty(String name) { + return (getProperty(name) != null); + } + + /** + * Return the value associated with the given name, or {@code null} if not + * found. + * + * @param name the property to find + */ + public abstract Object getProperty(String name); + + /** + * This {@code PropertySource} object is equal to the given object if: + *

    + *
  • they are the same instance + *
  • the {@code name} properties for both objects are equal + *
+ *

+ * No properties other than {@code name} are evaluated. + */ + @Override + public boolean equals(Object obj) { + return (this == obj || (obj instanceof PropertySource + && ObjectUtils.nullSafeEquals(this.name, ((PropertySource) obj).name))); + } + + /** + * Return a hash code derived from the {@code name} property of this + * {@code PropertySource} object. + */ + @Override + public int hashCode() { + return ObjectUtils.nullSafeHashCode(this.name); + } + + /** + * Produce concise output (type and name) if the current log level does not + * include debug. If debug is enabled, produce verbose output including the hash + * code of the PropertySource instance and every name/value property pair. + *

+ * This variable verbosity is useful as a property source such as system + * properties or environment variables may contain an arbitrary number of + * property pairs, potentially leading to difficult to read exception and log + * messages. + * + * @see Log#isDebugEnabled() + */ + @Override + public String toString() { + if (logger.isDebugEnabled()) { + return getClass().getSimpleName() + "@" + System.identityHashCode(this) + " {name='" + this.name + + "', properties=" + this.source + "}"; + } else { + return getClass().getSimpleName() + " {name='" + this.name + "'}"; + } + } + + /** + * Return a {@code PropertySource} implementation intended for collection + * comparison purposes only. + *

+ * Primarily for internal use, but given a collection of {@code PropertySource} + * objects, may be used as follows: + * + * 

+     *
+     * {
+     *     @code
+     *     List> sources = new ArrayList>();
+     *     sources.add(new MapPropertySource("sourceA", mapA));
+     *     sources.add(new MapPropertySource("sourceB", mapB));
+     *     assert sources.contains(PropertySource.named("sourceA"));
+     *     assert sources.contains(PropertySource.named("sourceB"));
+     *     assert !sources.contains(PropertySource.named("sourceC"));
+     * }
+     *
+ * + * The returned {@code PropertySource} will throw + * {@code UnsupportedOperationException} if any methods other than + * {@code equals(Object)}, {@code hashCode()}, and {@code toString()} are + * called. + * + * @param name the name of the comparison {@code PropertySource} to be created + * and returned. + */ + public static PropertySource named(String name) { + return new ComparisonPropertySource(name); + } + + /** + * {@code PropertySource} to be used as a placeholder in cases where an actual + * property source cannot be eagerly initialized at application context creation + * time. For example, a {@code ServletContext}-based property source must wait + * until the {@code ServletContext} object is available to its enclosing + * {@code ApplicationContext}. In such cases, a stub should be used to hold the + * intended default position/order of the property source, then be replaced + * during context refresh. + * + * @see org.springframework.web.context.support.StandardServletEnvironment + * @see org.springframework.web.context.support.ServletContextPropertySource + */ + public static class StubPropertySource extends PropertySource { + + public StubPropertySource(String name){ + super(name, new Object()); + } + + /** + * Always returns {@code null}. + */ + @Override + public String getProperty(String name) { + return null; + } + } + + /** + * @see PropertySource#named(String) + */ + static class ComparisonPropertySource extends StubPropertySource { + + private static final String USAGE_ERROR = "ComparisonPropertySource instances are for use with collection comparison only"; + + public ComparisonPropertySource(String name){ + super(name); + } + + @Override + public Object getSource() { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public boolean containsProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public String getProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java new file mode 100644 index 00000000..982f61b8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; + +import org.springframework.core.io.support.SpringFactoriesLoader; + +/** + * Strategy interface located via {@link SpringFactoriesLoader} and used to load + * a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + */ +public interface PropertySourceLoader { + + /** + * Returns the file extensions that the loader supports (excluding the '.'). + * + * @return the file extensions + */ + String[] getFileExtensions(); + + /** + * Load the resource into a property source. + * + * @param name the name of the property source + * @param resource the resource to load + * @param profile the name of the profile to load or {@code null}. The profile + * can be used to load multi-document files (such as YAML). Simple property + * formats should {@code null} when asked to load a profile. + * @return a property source or {@code null} + * @throws IOException if the source cannot be loaded + */ + PropertySource load(String name, Resource resource, String profile) throws IOException; +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java new file mode 100644 index 00000000..3cd08c9a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java @@ -0,0 +1,25 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +/** + * Holder containing one or more {@link PropertySource} objects. + * + * @author Chris Beams + * @since 3.1 + */ +public interface PropertySources extends Iterable> { + + /** + * Return whether a property source with the given name is contained. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + boolean contains(String name); + + /** + * Return the property source with the given name, {@code null} if not found. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + PropertySource get(String name); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java new file mode 100644 index 00000000..88301f20 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URL; + +import org.springframework.core.io.*; +import org.springframework.core.io.ByteArrayResource; + +/** + * Interface for a resource descriptor that abstracts from the actual type of + * underlying resource, such as a file or class path resource. + *

+ * An InputStream can be opened for every resource if it exists in physical + * form, but a URL or File handle can just be returned for certain resources. + * The actual behavior is implementation-specific. + * + * @author Juergen Hoeller + * @since 28.12.2003 + * @see #getInputStream() + * @see #getURL() + * @see #getURI() + * @see #getFile() + * @see WritableResource + * @see ContextResource + * @see UrlResource + * @see ClassPathResource + * @see FileSystemResource + * @see PathResource + * @see ByteArrayResource + * @see InputStreamResource + */ +public interface Resource extends InputStreamSource { + + boolean exists(); + + boolean isReadable(); + + boolean isOpen(); + + URL getURL() throws IOException; + + URI getURI() throws IOException; + + File getFile() throws IOException; + + long contentLength() throws IOException; + + long lastModified() throws IOException; + + org.springframework.core.io.Resource createRelative(String var1) throws IOException; + + String getFilename(); + + String getDescription(); +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java new file mode 100644 index 00000000..f59c6983 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java @@ -0,0 +1,182 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.core.env.Environment; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * {@link YamlProcessor.DocumentMatcher} backed by + * {@link Environment#getActiveProfiles()}. A YAML document may define a + * "spring.profiles" element as a comma-separated list of Spring profile names, + * optionally negated using the {@code !} character. If both negated and + * non-negated profiles are specified for a single document, at least one + * non-negated profile must match and no negated profiles may match. + * + * @author Dave Syer + * @author Matt Benson + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatcher { + + private String[] activeProfiles = new String[0]; + + public SpringProfileDocumentMatcher(){ + } + + public SpringProfileDocumentMatcher(String... profiles){ + addActiveProfiles(profiles); + } + + public void addActiveProfiles(String... profiles) { + LinkedHashSet set = new LinkedHashSet(Arrays.asList(this.activeProfiles)); + Collections.addAll(set, profiles); + this.activeProfiles = set.toArray(new String[set.size()]); + } + + @Override + public YamlProcessor.MatchStatus matches(Properties properties) { + List profiles = extractSpringProfiles(properties); + ProfilesMatcher profilesMatcher = getProfilesMatcher(); + Set negative = extractProfiles(profiles, ProfileType.NEGATIVE); + Set positive = extractProfiles(profiles, ProfileType.POSITIVE); + if (!CollectionUtils.isEmpty(negative)) { + if (profilesMatcher.matches(negative) == YamlProcessor.MatchStatus.FOUND) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + if (CollectionUtils.isEmpty(positive)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return profilesMatcher.matches(positive); + } + + private List extractSpringProfiles(Properties properties) { + SpringProperties springProperties = new SpringProperties(); + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(new PropertiesPropertySource("profiles", properties)); + // PropertyValues propertyValues = new PropertySourcesPropertyValues( + // propertySources); + // new RelaxedDataBinder(springProperties, "spring").bind(propertyValues); + // TODO + List profiles = springProperties.getProfiles(); + return profiles; + } + + private ProfilesMatcher getProfilesMatcher() { + return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher( + new HashSet(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher()); + } + + private Set extractProfiles(List profiles, ProfileType type) { + if (CollectionUtils.isEmpty(profiles)) { + return null; + } + Set extractedProfiles = new HashSet(); + for (String candidate : profiles) { + ProfileType candidateType = ProfileType.POSITIVE; + if (candidate.startsWith("!")) { + candidateType = ProfileType.NEGATIVE; + } + if (candidateType == type) { + extractedProfiles.add(type != ProfileType.POSITIVE ? candidate.substring(1) : candidate); + } + } + return extractedProfiles; + } + + /** + * Profile match types. + */ + enum ProfileType { + + POSITIVE, NEGATIVE + + } + + /** + * Base class for profile matchers. + */ + private abstract static class ProfilesMatcher { + + public final YamlProcessor.MatchStatus matches(Set profiles) { + if (CollectionUtils.isEmpty(profiles)) { + return YamlProcessor.MatchStatus.ABSTAIN; + } + return doMatches(profiles); + } + + protected abstract YamlProcessor.MatchStatus doMatches(Set profiles); + + } + + /** + * {@link ProfilesMatcher} that matches when a value in {@code spring.profiles} + * is also in {@code spring.profiles.active}. + */ + private static class ActiveProfilesMatcher extends ProfilesMatcher { + + private final Set activeProfiles; + + ActiveProfilesMatcher(Set activeProfiles){ + this.activeProfiles = activeProfiles; + } + + @Override + protected YamlProcessor.MatchStatus doMatches(Set profiles) { + if (profiles.isEmpty()) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + for (String activeProfile : this.activeProfiles) { + if (profiles.contains(activeProfile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * {@link ProfilesMatcher} that matches when {@code + * spring.profiles} is empty or contains a value with no text. + * + * @see StringUtils#hasText(String) + */ + private static class EmptyProfilesMatcher extends ProfilesMatcher { + + @Override + public YamlProcessor.MatchStatus doMatches(Set springProfiles) { + if (springProfiles.isEmpty()) { + return YamlProcessor.MatchStatus.FOUND; + } + for (String profile : springProfiles) { + if (!StringUtils.hasText(profile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * Class for binding {@code spring.profiles} property. + */ + static class SpringProperties { + + private List profiles = new ArrayList(); + + public List getProfiles() { + return this.profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java new file mode 100644 index 00000000..819c99fc --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java @@ -0,0 +1,419 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.io.Reader; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.parser.ParserException; +import org.yaml.snakeyaml.reader.UnicodeReader; + +/** + * Base class for YAML factories. + * + * @author Dave Syer + * @author Juergen Hoeller + * @since 4.1 + */ +public abstract class YamlProcessor { + + private final Log logger = LogFactory.getLog(getClass()); + + private ResolutionMethod resolutionMethod = ResolutionMethod.OVERRIDE; + + private Resource[] resources = new Resource[0]; + + private List documentMatchers = Collections.emptyList(); + + private boolean matchDefault = true; + + /** + * A map of document matchers allowing callers to selectively use only some of + * the documents in a YAML resource. In YAML documents are separated by + * --- lines, and each document is converted to properties before + * the match is made. E.g. + * + * 

+     * environment: dev
+     * url: http://dev.bar.com
+     * name: Developer Setup
+     * ---
+     * environment: prod
+     * url:http://foo.bar.com
+     * name: My Cool App
+     *
+ * + * when mapped with + * + * 
+     * setDocumentMatchers(properties -> ("prod"
+     *     .equals(properties.getProperty("environment")) ? MatchStatus.FOUND : MatchStatus.NOT_FOUND));
+     *
+ * + * would end up as + * + * 
+     * environment=prod
+     * url=http://foo.bar.com
+     * name=My Cool App
+     *
+ */ + public void setDocumentMatchers(DocumentMatcher... matchers) { + this.documentMatchers = Arrays.asList(matchers); + } + + /** + * Flag indicating that a document for which all the + * {@link #setDocumentMatchers(DocumentMatcher...) document matchers} abstain + * will nevertheless match. Default is {@code true}. + */ + public void setMatchDefault(boolean matchDefault) { + this.matchDefault = matchDefault; + } + + /** + * Method to use for resolving resources. Each resource will be converted to a + * Map, so this property is used to decide which map entries to keep in the + * final output from this factory. Default is {@link ResolutionMethod#OVERRIDE}. + */ + public void setResolutionMethod(ResolutionMethod resolutionMethod) { + Assert.notNull(resolutionMethod, "ResolutionMethod must not be null"); + this.resolutionMethod = resolutionMethod; + } + + /** + * Set locations of YAML {@link Resource resources} to be loaded. + * + * @see ResolutionMethod + */ + public void setResources(Resource... resources) { + this.resources = resources; + } + + /** + * Provide an opportunity for subclasses to process the Yaml parsed from the + * supplied resources. Each resource is parsed in turn and the documents inside + * checked against the {@link #setDocumentMatchers(DocumentMatcher...) + * matchers}. If a document matches it is passed into the callback, along with + * its representation as Properties. Depending on the + * {@link #setResolutionMethod(ResolutionMethod)} not all of the documents will + * be parsed. + * + * @param callback a callback to delegate to once matching documents are found + * @see #createYaml() + */ + protected void process(MatchCallback callback) { + Yaml yaml = createYaml(); + for (Resource resource : this.resources) { + boolean found = process(callback, yaml, resource); + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND && found) { + return; + } + } + } + + /** + * Create the {@link Yaml} instance to use. + */ + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor()); + } + + private boolean process(MatchCallback callback, Yaml yaml, Resource resource) { + int count = 0; + try { + if (logger.isDebugEnabled()) { + logger.debug("Loading from YAML: " + resource); + } + Reader reader = new UnicodeReader(resource.getInputStream()); + try { + for (Object object : yaml.loadAll(reader)) { + if (object != null && process(asMap(object), callback)) { + count++; + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND) { + break; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug( + "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource); + } + } finally { + reader.close(); + } + } catch (IOException ex) { + handleProcessError(resource, ex); + } + return (count > 0); + } + + private void handleProcessError(Resource resource, IOException ex) { + if (this.resolutionMethod != ResolutionMethod.FIRST_FOUND + && this.resolutionMethod != ResolutionMethod.OVERRIDE_AND_IGNORE) { + throw new IllegalStateException(ex); + } + if (logger.isWarnEnabled()) { + logger.warn("Could not load map from " + resource + ": " + ex.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private Map asMap(Object object) { + // YAML can have numbers as keys + Map result = new LinkedHashMap(); + if (!(object instanceof Map)) { + // A document can be a text literal + result.put("document", object); + return result; + } + + Map map = (Map) object; + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Map) { + value = asMap(value); + } + Object key = entry.getKey(); + if (key instanceof CharSequence) { + result.put(key.toString(), value); + } else { + // It has to be a map key in this case + result.put("[" + key.toString() + "]", value); + } + } + return result; + } + + private boolean process(Map map, MatchCallback callback) { + Properties properties = new Properties() { + + @Override + public String getProperty(String key) { + Object value = get(key); + return (value != null ? value.toString() : null); + } + }; + properties.putAll(getFlattenedMap(map)); + + if (this.documentMatchers.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Merging document (no matchers set): " + map); + } + callback.process(properties, map); + return true; + } + + MatchStatus result = MatchStatus.ABSTAIN; + for (DocumentMatcher matcher : this.documentMatchers) { + MatchStatus match = matcher.matches(properties); + result = MatchStatus.getMostSpecific(match, result); + if (match == MatchStatus.FOUND) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with document matcher: " + properties); + } + callback.process(properties, map); + return true; + } + } + + if (result == MatchStatus.ABSTAIN && this.matchDefault) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with default matcher: " + map); + } + callback.process(properties, map); + return true; + } + + if (logger.isDebugEnabled()) { + logger.debug("Unmatched document: " + map); + } + return false; + } + + /** + * Return a flattened version of the given map, recursively following any nested + * Map or Collection values. Entries from the resulting map retain the same + * order as the source. When called with the Map from a {@link MatchCallback} + * the result will contain the same values as the {@link MatchCallback} + * Properties. + * + * @param source the source map + * @return a flattened map + * @since 4.1.3 + */ + protected final Map getFlattenedMap(Map source) { + Map result = new LinkedHashMap(); + buildFlattenedMap(result, source, null); + return result; + } + + private static boolean containsText(CharSequence str) { + int strLen = str.length(); + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + return false; + } + + private void buildFlattenedMap(Map result, Map source, String path) { + for (Map.Entry entry : source.entrySet()) { + String key = entry.getKey(); + if (path != null && !path.isEmpty() && containsText(path)) { + if (key.startsWith("[")) { + key = path + key; + } else { + key = path + '.' + key; + } + } + Object value = entry.getValue(); + if (value instanceof String) { + result.put(key, value); + } else if (value instanceof Map) { + // Need a compound key + @SuppressWarnings("unchecked") + Map map = (Map) value; + buildFlattenedMap(result, map, key); + } else if (value instanceof Collection) { + // Need a compound key + @SuppressWarnings("unchecked") + Collection collection = (Collection) value; + int count = 0; + for (Object object : collection) { + buildFlattenedMap(result, Collections.singletonMap("[" + (count++) + "]", object), key); + } + } else { + result.put(key, (value != null ? value : "")); + } + } + } + + /** + * Callback interface used to process the YAML parsing results. + */ + public interface MatchCallback { + + /** + * Process the given representation of the parsing results. + * + * @param properties the properties to process (as a flattened representation + * with indexed keys in case of a collection or map) + * @param map the result map (preserving the original value structure in the + * YAML document) + */ + void process(Properties properties, Map map); + } + + /** + * Strategy interface used to test if properties match. + */ + public interface DocumentMatcher { + + /** + * Test if the given properties match. + * + * @param properties the properties to test + * @return the status of the match + */ + MatchStatus matches(Properties properties); + } + + /** + * Status returned from {@link DocumentMatcher#matches(Properties)} + */ + public enum MatchStatus { + + /** + * A match was found. + */ + FOUND, + + /** + * No match was found. + */ + NOT_FOUND, + + /** + * The matcher should not be considered. + */ + ABSTAIN; + + /** + * Compare two {@link MatchStatus} items, returning the most specific status. + */ + public static MatchStatus getMostSpecific(MatchStatus a, MatchStatus b) { + return (a.ordinal() < b.ordinal() ? a : b); + } + } + + /** + * Method to use for resolving resources. + */ + public enum ResolutionMethod { + + /** + * Replace values from earlier in the list. + */ + OVERRIDE, + + /** + * Replace values from earlier in the list, ignoring any failures. + */ + OVERRIDE_AND_IGNORE, + + /** + * Take the first resource in the list that exists and use just that. + */ + FIRST_FOUND + } + + /** + * A specialized {@link Constructor} that checks for duplicate keys. + */ + protected static class StrictMapAppenderConstructor extends Constructor { + + // Declared as public for use in subclasses + public StrictMapAppenderConstructor(){ + super(); + } + + @Override + protected Map constructMapping(MappingNode node) { + try { + return super.constructMapping(node); + } catch (IllegalStateException ex) { + throw new ParserException("while parsing MappingNode", + node.getStartMark(), + ex.getMessage(), + node.getEndMark()); + } + } + + @Override + protected Map createDefaultMap() { + final Map delegate = super.createDefaultMap(); + return new AbstractMap() { + + @Override + public Object put(Object key, Object value) { + if (delegate.containsKey(key)) { + throw new IllegalStateException("Duplicate key: " + key); + } + return delegate.put(key, value); + } + + @Override + public Set> entrySet() { + return delegate.entrySet(); + } + }; + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java new file mode 100644 index 00000000..936bdba6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java @@ -0,0 +1,87 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Pattern; + +import org.springframework.util.ClassUtils; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; +import org.yaml.snakeyaml.resolver.Resolver; + +/** + * Strategy to load '.yml' (or '.yaml') files into a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class YamlPropertySourceLoader implements PropertySourceLoader { + + @Override + public String[] getFileExtensions() { + return new String[] { "yml", "yaml" }; + } + + @Override + public PropertySource load(String name, Resource resource, String profile) throws IOException { + if (ClassUtils.isPresent("org.yaml.snakeyaml.Yaml", null)) { + Processor processor = new Processor(resource, profile); + Map source = processor.process(); + if (!source.isEmpty()) { + return new MapPropertySource(name, source); + } + } + return null; + } + + /** + * {@link YamlProcessor} to create a {@link Map} containing the property values. + * Similar to {@link YamlPropertiesFactoryBean} but retains the order of + * entries. + */ + private static class Processor extends YamlProcessor { + + Processor(Resource resource, String profile){ + if (profile == null) { + setMatchDefault(true); + setDocumentMatchers(new SpringProfileDocumentMatcher()); + } else { + setMatchDefault(false); + setDocumentMatchers(new SpringProfileDocumentMatcher(profile)); + } + setResources(resource); + } + + @Override + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor(), new Representer(), new DumperOptions(), new Resolver() { + + @Override + public void addImplicitResolver(Tag tag, Pattern regexp, String first) { + if (tag == Tag.TIMESTAMP) { + return; + } + super.addImplicitResolver(tag, regexp, first); + } + }); + } + + public Map process() { + final Map result = new LinkedHashMap(); + process(new MatchCallback() { + + @Override + public void process(Properties properties, Map map) { + result.putAll(getFlattenedMap(map)); + } + }); + return result; + } + + } +} diff --git a/client-adapter/elasticsearch/pom.xml b/client-adapter/elasticsearch/pom.xml index 35711014..90c64f80 100644 --- a/client-adapter/elasticsearch/pom.xml +++ b/client-adapter/elasticsearch/pom.xml @@ -18,12 +18,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.19 - provided - com.alibaba.fastsql fastsql @@ -92,4 +86,4 @@ - \ No newline at end of file + diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index a892cf04..b22ecdca 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -1,10 +1,7 @@ package com.alibaba.otter.canal.client.adapter.es; import java.net.InetAddress; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,11 +25,7 @@ import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor; import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.SPI; +import com.alibaba.otter.canal.client.adapter.support.*; /** * ES外部适配器 @@ -69,9 +62,9 @@ public class ESAdapter implements OuterAdapter { } @Override - public void init(OuterAdapterConfig configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { try { - Map esSyncConfigTmp = ESSyncConfigLoader.load(); + Map esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); // 过滤不匹配的key的配置 esSyncConfigTmp.forEach((key, config) -> { if ((config.getOuterAdapterKey() == null && configuration.getKey() == null) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index fcf94105..d721cc3d 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -2,12 +2,12 @@ package com.alibaba.otter.canal.client.adapter.es.config; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -20,17 +20,15 @@ public class ESSyncConfigLoader { private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); - @SuppressWarnings("unchecked") - public static synchronized Map load() { + public static synchronized Map load(Properties envProperties) { logger.info("## Start loading es mapping config ... "); Map esSyncConfig = new LinkedHashMap<>(); Map configContentMap = MappingConfigsLoader.loadConfigs("es"); configContentMap.forEach((fileName, content) -> { - Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定 - JSONObject configJson = new JSONObject(configMap); - ESSyncConfig config = configJson.toJavaObject(ESSyncConfig.class); + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties); + try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java index 7ab197d1..a66acbb9 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -21,7 +21,7 @@ public class ConfigLoadTest { @Test public void testLoad() { - Map configMap = ESSyncConfigLoader.load(); + Map configMap = ESSyncConfigLoader.load(null); ESSyncConfig config = configMap.get("mytest_user.yml"); Assert.assertNotNull(config); Assert.assertEquals("defaultDS", config.getDataSourceKey()); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java index 10d41de1..3f9c10d4 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java @@ -26,7 +26,7 @@ public class Common { outerAdapterConfig.setProperties(properties); ESAdapter esAdapter = new ESAdapter(); - esAdapter.init(outerAdapterConfig); + esAdapter.init(outerAdapterConfig, null); return esAdapter; } diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index 29ee8b33..87e61c92 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -17,12 +17,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.19 - provided - org.apache.hbase hbase-client diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 95d00ca8..45b1bd51 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -1,10 +1,7 @@ package com.alibaba.otter.canal.client.adapter.hbase; import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import javax.sql.DataSource; @@ -58,9 +55,9 @@ public class HbaseAdapter implements OuterAdapter { } @Override - public void init(OuterAdapterConfig configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { try { - Map hbaseMappingTmp = MappingConfigLoader.load(); + Map hbaseMappingTmp = MappingConfigLoader.load(envProperties); // 过滤不匹配的key的配置 hbaseMappingTmp.forEach((key, mappingConfig) -> { if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 3cf16a71..ff81e6b7 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -2,12 +2,12 @@ package com.alibaba.otter.canal.client.adapter.hbase.config; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -25,17 +25,15 @@ public class MappingConfigLoader { * * @return 配置名/配置文件名--对象 */ - @SuppressWarnings("unchecked") - public static Map load() { + public static Map load(Properties envProperties) { logger.info("## Start loading hbase mapping config ... "); Map result = new LinkedHashMap<>(); Map configContentMap = MappingConfigsLoader.loadConfigs("hbase"); configContentMap.forEach((fileName, content) -> { - Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定 - JSONObject configJson = new JSONObject(configMap); - MappingConfig config = configJson.toJavaObject(MappingConfig.class); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 80d5371a..7d082bc6 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -2,10 +2,7 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -13,7 +10,12 @@ import java.util.concurrent.Future; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; @@ -62,7 +64,7 @@ public class CanalAdapterLoader { for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { List canalOutConnectors = new ArrayList<>(); for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { - loadConnector(c, canalOutConnectors); + loadAdapter(c, canalOutConnectors); } canalOuterAdapterGroups.add(canalOutConnectors); } @@ -91,7 +93,7 @@ public class CanalAdapterLoader { List> canalOuterAdapterGroups = new ArrayList<>(); List canalOuterAdapters = new ArrayList<>(); for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadConnector(config, canalOuterAdapters); + loadAdapter(config, canalOuterAdapters); } canalOuterAdapterGroups.add(canalOuterAdapters); @@ -103,8 +105,8 @@ public class CanalAdapterLoader { canalClientConfig.getFlatMessage()); canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); canalKafkaWorker.start(); - logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-" - + group.getGroupId()); + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); } } } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { @@ -114,7 +116,7 @@ public class CanalAdapterLoader { List> canalOuterAdapterGroups = new ArrayList<>(); List canalOuterAdapters = new ArrayList<>(); for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadConnector(config, canalOuterAdapters); + loadAdapter(config, canalOuterAdapters); } canalOuterAdapterGroups.add(canalOuterAdapters); CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, @@ -128,14 +130,14 @@ public class CanalAdapterLoader { canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); rocketMQWorker.start(); - logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-" - + group.getGroupId()); + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); } } } } - private void loadConnector(OuterAdapterConfig config, List canalOutConnectors) { + private void loadAdapter(OuterAdapterConfig config, List canalOutConnectors) { try { OuterAdapter adapter; adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey())); @@ -143,7 +145,23 @@ public class CanalAdapterLoader { ClassLoader cl = Thread.currentThread().getContextClassLoader(); // 替换ClassLoader Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); - adapter.init(config); + Environment env = (Environment) SpringContext.getBean(Environment.class); + Properties evnProperties = null; + if (env instanceof StandardEnvironment) { + evnProperties = new Properties(); + for (PropertySource propertySource : ((StandardEnvironment) env).getPropertySources()) { + if (propertySource instanceof EnumerablePropertySource) { + String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames(); + for (String name : names) { + Object val = propertySource.getProperty(name); + if (val != null) { + evnProperties.put(name, val); + } + } + } + } + } + adapter.init(config, evnProperties); Thread.currentThread().setContextClassLoader(cl); canalOutConnectors.add(adapter); logger.info("Load canal adapter: {} succeed", config.getName()); diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index bd7c87ab..08fd171f 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -1,6 +1,7 @@ package com.alibaba.otter.canal.client.adapter.logger; import java.util.List; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +26,7 @@ public class LoggerAdapterExample implements OuterAdapter { private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void init(OuterAdapterConfig configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { } diff --git a/client-adapter/rdb/pom.xml b/client-adapter/rdb/pom.xml index 211950e9..0fb2c0ea 100644 --- a/client-adapter/rdb/pom.xml +++ b/client-adapter/rdb/pom.xml @@ -18,13 +18,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.19 - provided - - mysql mysql-connector-java diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java index 2df46974..da83c6c7 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java @@ -5,10 +5,8 @@ import java.sql.SQLException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import javax.sql.DataSource; @@ -27,12 +25,7 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.SPI; -import com.alibaba.otter.canal.client.adapter.support.Util; +import com.alibaba.otter.canal.client.adapter.support.*; /** * RDB适配器实现类 @@ -74,8 +67,8 @@ public class RdbAdapter implements OuterAdapter { * @param configuration 外部适配器配置信息 */ @Override - public void init(OuterAdapterConfig configuration) { - Map rdbMappingTmp = ConfigLoader.load(); + public void init(OuterAdapterConfig configuration, Properties envProperties) { + Map rdbMappingTmp = ConfigLoader.load(envProperties); // 过滤不匹配的key的配置 rdbMappingTmp.forEach((key, mappingConfig) -> { if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java index 8d249733..63dd728a 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java @@ -2,8 +2,10 @@ package com.alibaba.otter.canal.client.adapter.rdb.config; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; import com.alibaba.fastjson.JSONObject; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; @@ -25,17 +27,15 @@ public class ConfigLoader { * * @return 配置名/配置文件名--对象 */ - @SuppressWarnings("unchecked") - public static Map load() { + public static Map load(Properties envProperties) { logger.info("## Start loading rdb mapping config ... "); Map result = new LinkedHashMap<>(); Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); configContentMap.forEach((fileName, content) -> { - Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定 - JSONObject configJson = new JSONObject(configMap); - MappingConfig config = configJson.toJavaObject(MappingConfig.class); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java index 5eeb0878..6fbf3fd3 100644 --- a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java @@ -20,7 +20,7 @@ public class ConfigLoadTest { @Test public void testLoad() { - Map configMap = ConfigLoader.load(); + Map configMap = ConfigLoader.load(null); Assert.assertFalse(configMap.isEmpty()); } diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java index 9d2d117d..de949147 100644 --- a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java @@ -24,7 +24,7 @@ public class Common { outerAdapterConfig.setProperties(properties); RdbAdapter adapter = new RdbAdapter(); - adapter.init(outerAdapterConfig); + adapter.init(outerAdapterConfig, null); return adapter; } } From 597a08ac37085e369293f98ba60cf6d220dbe2a0 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 1 Feb 2019 13:51:49 +0800 Subject: [PATCH 03/10] =?UTF-8?q?=E9=80=9A=E8=BF=87groupId=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=E5=AF=B9=E5=BA=94=E7=9A=84adapter=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/support/Dml.java | 9 +++ .../client/adapter/support/MessageUtil.java | 10 +-- .../canal/client/adapter/es/ESAdapter.java | 15 ++++- .../adapter/es/config/ESSyncConfig.java | 10 +++ .../client/adapter/hbase/HbaseAdapter.java | 14 +++- .../adapter/hbase/config/MappingConfig.java | 10 +++ .../loader/AbstractCanalAdapterWorker.java | 5 +- .../loader/CanalAdapterKafkaWorker.java | 7 +- .../loader/CanalAdapterRocketMQWorker.java | 3 +- .../adapter/rdb/config/MappingConfig.java | 10 +++ .../adapter/rdb/config/MirrorDbConfig.java | 2 +- .../rdb/service/RdbMirrorDbSyncService.java | 66 +++++++++++-------- .../adapter/rdb/service/RdbSyncService.java | 16 ++++- 13 files changed, 135 insertions(+), 42 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java index 38fd7a3b..97204867 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/Dml.java @@ -15,6 +15,7 @@ public class Dml implements Serializable { private static final long serialVersionUID = 2611556444074013268L; private String destination; // 对应canal的实例或者MQ的topic + private String groupId; // 对应mq的group id private String database; // 数据库或schema private String table; // 表名 private List pkNames; @@ -36,6 +37,14 @@ public class Dml implements Serializable { this.destination = destination; } + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + public String getDatabase() { return database; } diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java index 1ddbae0f..239b8839 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java @@ -14,7 +14,7 @@ import com.alibaba.otter.canal.protocol.Message; */ public class MessageUtil { - public static List parse4Dml(String destination, Message message) { + public static List parse4Dml(String destination, String groupId, Message message) { if (message == null) { return null; } @@ -38,6 +38,7 @@ public class MessageUtil { final Dml dml = new Dml(); dml.setDestination(destination); + dml.setGroupId(groupId); dml.setDatabase(entry.getHeader().getSchemaName()); dml.setTable(entry.getHeader().getTableName()); dml.setType(eventType.toString()); @@ -118,10 +119,10 @@ public class MessageUtil { return dmls; } - public static List flatMessage2Dml(String destination, List flatMessages) { + public static List flatMessage2Dml(String destination, String groupId, List flatMessages) { List dmls = new ArrayList(flatMessages.size()); for (FlatMessage flatMessage : flatMessages) { - Dml dml = flatMessage2Dml(destination, flatMessage); + Dml dml = flatMessage2Dml(destination, groupId, flatMessage); if (dml != null) { dmls.add(dml); } @@ -130,12 +131,13 @@ public class MessageUtil { return dmls; } - public static Dml flatMessage2Dml(String destination, FlatMessage flatMessage) { + public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) { if (flatMessage == null) { return null; } Dml dml = new Dml(); dml.setDestination(destination); + dml.setGroupId(groupId); dml.setDatabase(flatMessage.getDatabase()); dml.setTable(flatMessage.getTable()); dml.setPkNames(flatMessage.getPkNames()); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index b22ecdca..ab57c1d8 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -8,6 +8,7 @@ import java.util.regex.Pattern; import javax.sql.DataSource; +import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; @@ -133,7 +134,19 @@ public class ESAdapter implements OuterAdapter { String table = dml.getTable(); Map configMap = dbTableEsSyncConfig.get(database + "-" + table); if (configMap != null) { - esSyncService.sync(configMap.values(), dml); + List configs = new ArrayList<>(); + configMap.values().forEach(esConfig -> { + if (StringUtils.isNotEmpty(esConfig.getGroupId())) { + if (esConfig.getGroupId().equals(dml.getGroupId())) { + configs.add(esConfig); + } + } else { + configs.add(esConfig); + } + }); + if (!configs.isEmpty()) { + esSyncService.sync(configs, dml); + } } } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java index 65087437..09c2f4e0 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java @@ -17,6 +17,8 @@ public class ESSyncConfig { private String outerAdapterKey; // adapter key + private String groupId; // group id + private String destination; // canal destination private ESMapping esMapping; @@ -52,6 +54,14 @@ public class ESSyncConfig { this.outerAdapterKey = outerAdapterKey; } + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + public String getDestination() { return destination; } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 45b1bd51..b54f8f46 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -108,7 +108,19 @@ public class HbaseAdapter implements OuterAdapter { String table = dml.getTable(); Map configMap = mappingConfigCache.get(destination + "." + database + "." + table); if (configMap != null) { - configMap.values().forEach(config -> hbaseSyncService.sync(config, dml)); + List configs = new ArrayList<>(); + configMap.values().forEach(config -> { + if (StringUtils.isNotEmpty(config.getGroupId())) { + if (config.getGroupId().equals(dml.getGroupId())) { + configs.add(config); + } + } else { + configs.add(config); + } + }); + if (!configs.isEmpty()) { + configs.forEach(config -> hbaseSyncService.sync(config, dml)); + } } } diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java index 2d86adee..3d1c38be 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java @@ -14,6 +14,8 @@ public class MappingConfig { private String outerAdapterKey; // adapter key + private String groupId; // groupId + private String destination; // canal实例或MQ的topic private HbaseMapping hbaseMapping; // hbase映射配置 @@ -26,6 +28,14 @@ public class MappingConfig { this.dataSourceKey = dataSourceKey; } + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + public String getOuterAdapterKey() { return outerAdapterKey; } diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java index e4d58272..6f21ebd5 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java @@ -31,6 +31,7 @@ public abstract class AbstractCanalAdapterWorker { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); protected String canalDestination; // canal实例 + protected String groupId = null; // groupId protected List> canalOuterAdapters; // 外部适配器 protected CanalClientConfig canalClientConfig; // 配置 protected ExecutorService groupInnerExecutorService; // 组内工作线程池 @@ -56,7 +57,7 @@ public abstract class AbstractCanalAdapterWorker { // 组内适配器穿行运行，尽量不要配置组内适配器 adapters.forEach(adapter -> { long begin = System.currentTimeMillis(); - List dmls = MessageUtil.parse4Dml(canalDestination, message); + List dmls = MessageUtil.parse4Dml(canalDestination, groupId, message); if (dmls != null) { batchSync(dmls, adapter); @@ -101,7 +102,7 @@ public abstract class AbstractCanalAdapterWorker { // 组内适配器穿行运行，尽量不要配置组内适配器 outerAdapters.forEach(adapter -> { long begin = System.currentTimeMillis(); - List dmls = MessageUtil.flatMessage2Dml(canalDestination, flatMessages); + List dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages); batchSync(dmls, adapter); if (logger.isDebugEnabled()) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java index 58ffed8f..72291ccb 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java @@ -28,7 +28,8 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { super(canalOuterAdapters); this.canalClientConfig = canalClientConfig; this.topic = topic; - this.canalDestination = topic; + super.canalDestination = topic; + super.groupId = groupId; this.flatMessage = flatMessage; this.connector = new KafkaCanalConnector(bootstrapServers, topic, @@ -45,10 +46,12 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker { try { Thread.sleep(1000); } catch (InterruptedException e) { + // ignore } } ExecutorService workerExecutor = Executors.newSingleThreadExecutor(); - int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); + int retry = canalClientConfig.getRetries() == null + || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries(); long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒 while (running) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java index b6f276b2..369e8965 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java @@ -28,7 +28,8 @@ public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker { this.canalClientConfig = canalClientConfig; this.topic = topic; this.flatMessage = flatMessage; - this.canalDestination = topic; + super.canalDestination = topic; + super.groupId = groupId; this.connector = new RocketMQCanalConnector(nameServers, topic, groupId, diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java index 5c0e9f9f..d2421020 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MappingConfig.java @@ -17,6 +17,8 @@ public class MappingConfig { private String destination; // canal实例或MQ的topic + private String groupId; // groupId + private String outerAdapterKey; // 对应适配器的key private Boolean concurrent; // 是否并行同步 @@ -31,6 +33,14 @@ public class MappingConfig { this.dataSourceKey = dataSourceKey; } + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + public String getOuterAdapterKey() { return outerAdapterKey; } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java index d84f66e7..dbead72e 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/MirrorDbConfig.java @@ -7,7 +7,7 @@ public class MirrorDbConfig { private String fileName; private MappingConfig mappingConfig; - Map tableConfig = new ConcurrentHashMap<>(); + private Map tableConfig = new ConcurrentHashMap<>(); public static MirrorDbConfig create(String fileName, MappingConfig mappingConfig) { return new MirrorDbConfig(fileName, mappingConfig); diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java index 16d8af5f..eec96e0c 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java @@ -56,6 +56,15 @@ public class RdbMirrorDbSyncService { if (mirrorDbConfig == null) { continue; } + if (mirrorDbConfig.getMappingConfig() == null) { + continue; + } + if (StringUtils.isNotEmpty(mirrorDbConfig.getMappingConfig().getGroupId())) { + if (!mirrorDbConfig.getMappingConfig().getGroupId().equals(dml.getGroupId())) { + continue; // 如果groupId不匹配则过滤 + } + } + if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { // DDL if (logger.isDebugEnabled()) { @@ -71,37 +80,35 @@ public class RdbMirrorDbSyncService { } } if (!dmlList.isEmpty()) { - rdbSyncService.sync(dmlList, - dml -> { - MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." - + dml.getDatabase()); - if (mirrorDbConfig == null) { - return false; - } - String table = dml.getTable(); - MappingConfig config = mirrorDbConfig.getTableConfig().get(table); + rdbSyncService.sync(dmlList, dml -> { + MirrorDbConfig mirrorDbConfig = mirrorDbConfigCache.get(dml.getDestination() + "." + dml.getDatabase()); + if (mirrorDbConfig == null) { + return false; + } + String table = dml.getTable(); + MappingConfig config = mirrorDbConfig.getTableConfig().get(table); - if (config == null) { - return false; - } + if (config == null) { + return false; + } - if (config.getConcurrent()) { - List singleDmls = SingleDml.dml2SingleDmls(dml); - singleDmls.forEach(singleDml -> { - int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData()); - RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); - rdbSyncService.getDmlsPartition()[hash].add(syncItem); - }); - } else { - int hash = 0; - List singleDmls = SingleDml.dml2SingleDmls(dml); - singleDmls.forEach(singleDml -> { - RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); - rdbSyncService.getDmlsPartition()[hash].add(syncItem); - }); - } - return true; - }); + if (config.getConcurrent()) { + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + int hash = rdbSyncService.pkHash(config.getDbMapping(), singleDml.getData()); + RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); + rdbSyncService.getDmlsPartition()[hash].add(syncItem); + }); + } else { + int hash = 0; + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + RdbSyncService.SyncItem syncItem = new RdbSyncService.SyncItem(config, singleDml); + rdbSyncService.getDmlsPartition()[hash].add(syncItem); + }); + } + return true; + }); } } @@ -119,6 +126,7 @@ public class RdbMirrorDbSyncService { mappingConfig = new MappingConfig(); mappingConfig.setDataSourceKey(baseConfigMap.getDataSourceKey()); mappingConfig.setDestination(baseConfigMap.getDestination()); + mappingConfig.setGroupId(baseConfigMap.getGroupId()); mappingConfig.setOuterAdapterKey(baseConfigMap.getOuterAdapterKey()); mappingConfig.setConcurrent(baseConfigMap.getConcurrent()); MappingConfig.DbMapping dbMapping = new MappingConfig.DbMapping(); diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java index d2ba7dac..68e79c8e 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java @@ -155,8 +155,22 @@ public class RdbSyncService { return false; } + List configs = new ArrayList<>(); + configMap.values().forEach(config -> { + if (StringUtils.isNotEmpty(config.getGroupId())) { + if (config.getGroupId().equals(dml.getGroupId())) { + configs.add(config); + } + } else { + configs.add(config); + } + }); + if (configs.isEmpty()) { + return false; + } + boolean executed = false; - for (MappingConfig config : configMap.values()) { + for (MappingConfig config : configs) { if (config.getConcurrent()) { List singleDmls = SingleDml.dml2SingleDmls(dml); singleDmls.forEach(singleDml -> { From 10276e254a6fcbd6069aff4ec90a062f88f4e4c4 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 1 Feb 2019 15:18:16 +0800 Subject: [PATCH 04/10] =?UTF-8?q?mq=E6=A8=A1=E5=BC=8F=E4=B8=8Badapter?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=A2=9E=E5=8A=A0groupId=E5=B1=9E=E6=80=A7?= =?UTF-8?q?=E5=AF=B9=E5=BA=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../canal/client/adapter/es/ESAdapter.java | 26 +++++++------------ .../src/main/resources/es/mytest_user.yml | 7 +++-- .../client/adapter/hbase/HbaseAdapter.java | 11 +++++--- .../main/resources/hbase/mytest_person2.yml | 3 ++- .../canal/client/adapter/rdb/RdbAdapter.java | 5 ++-- .../adapter/rdb/service/RdbSyncService.java | 8 +++--- .../src/main/resources/rdb/mytest_user.yml | 8 +++--- 7 files changed, 33 insertions(+), 35 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index ab57c1d8..9e54cce3 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -93,8 +93,10 @@ public class ESAdapter implements OuterAdapter { String schema = matcher.group(2); schemaItem.getAliasTableItems().values().forEach(tableItem -> { - Map esSyncConfigMap = dbTableEsSyncConfig - .computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new HashMap<>()); + Map esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent( + StringUtils.trimToEmpty(config.getDestination()) + "-" + StringUtils + .trimToEmpty(config.getGroupId()) + "_" + schema + "-" + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); esSyncConfigMap.put(configName, config); }); } @@ -132,21 +134,11 @@ public class ESAdapter implements OuterAdapter { public void sync(Dml dml) { String database = dml.getDatabase(); String table = dml.getTable(); - Map configMap = dbTableEsSyncConfig.get(database + "-" + table); - if (configMap != null) { - List configs = new ArrayList<>(); - configMap.values().forEach(esConfig -> { - if (StringUtils.isNotEmpty(esConfig.getGroupId())) { - if (esConfig.getGroupId().equals(dml.getGroupId())) { - configs.add(esConfig); - } - } else { - configs.add(esConfig); - } - }); - if (!configs.isEmpty()) { - esSyncService.sync(configs, dml); - } + Map configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + "_" + + database + "-" + table); + if (configMap != null && !configMap.values().isEmpty()) { + esSyncService.sync(configMap.values(), dml); } } diff --git a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml index 849420a3..9585227e 100644 --- a/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml +++ b/client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml @@ -1,15 +1,14 @@ dataSourceKey: defaultDS destination: example +groupId: g1 esMapping: _index: mytest_user _type: _doc _id: _id # pk: id sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name, - a.c_time as _c_time, c.labels as _labels from user a - left join role b on b.id=a.role_id - left join (select user_id, group_concat(label order by id desc separator ';') as labels from label - group by user_id) c on c.user_id=a.id" + a.c_time as _c_time from user a + left join role b on b.id=a.role_id" # objFields: # _labels: array:; etlCondition: "where a.c_time>='{0}'" diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index b54f8f46..795631e0 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -69,10 +69,11 @@ public class HbaseAdapter implements OuterAdapter { for (Map.Entry entry : hbaseMapping.entrySet()) { String configName = entry.getKey(); MappingConfig mappingConfig = entry.getValue(); - String k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." - + mappingConfig.getHbaseMapping().getDatabase() + "." + String k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + + mappingConfig.getHbaseMapping().getDatabase() + "-" + mappingConfig.getHbaseMapping().getTable(); - Map configMap = mappingConfigCache.computeIfAbsent(k, k1 -> new HashMap<>()); + Map configMap = mappingConfigCache.computeIfAbsent(k, k1 -> new ConcurrentHashMap<>()); configMap.put(configName, mappingConfig); } @@ -104,9 +105,11 @@ public class HbaseAdapter implements OuterAdapter { return; } String destination = StringUtils.trimToEmpty(dml.getDestination()); + String groupId = StringUtils.trimToEmpty(dml.getGroupId()); String database = dml.getDatabase(); String table = dml.getTable(); - Map configMap = mappingConfigCache.get(destination + "." + database + "." + table); + Map configMap = mappingConfigCache + .get(destination + "-" + groupId + "_" + database + "-" + table); if (configMap != null) { List configs = new ArrayList<>(); configMap.values().forEach(config -> { diff --git a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml index 0eebad47..703894b3 100644 --- a/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml +++ b/client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml @@ -1,5 +1,6 @@ dataSourceKey: defaultDS destination: example +groupId: g1 hbaseMapping: mode: STRING #NATIVE #PHOENIX database: mytest # 数据库名 @@ -57,4 +58,4 @@ hbaseMapping: # $UNSIGNED_TIMESTAMP 对应PHOENIX里的UNSIGNED_TIMESTAMP 12字节 # $VARCHAR 对应PHOENIX里的VARCHAR 动态长度 # $VARBINARY 对应PHOENIX里的VARBINARY 动态长度 -# $DECIMAL 对应PHOENIX里的DECIMAL 动态长度 \ No newline at end of file +# $DECIMAL 对应PHOENIX里的DECIMAL 动态长度 diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java index da83c6c7..d06e03c1 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java @@ -86,8 +86,9 @@ public class RdbAdapter implements OuterAdapter { String configName = entry.getKey(); MappingConfig mappingConfig = entry.getValue(); if (!mappingConfig.getDbMapping().getMirrorDb()) { - String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." - + mappingConfig.getDbMapping().getDatabase() + "." + String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable(); Map configMap = mappingConfigCache.computeIfAbsent(key, k1 -> new ConcurrentHashMap<>()); diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java index 68e79c8e..bf8d2de2 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java @@ -147,9 +147,11 @@ public class RdbSyncService { } else { // DML String destination = StringUtils.trimToEmpty(dml.getDestination()); + String groupId = StringUtils.trimToEmpty(dml.getGroupId()); String database = dml.getDatabase(); String table = dml.getTable(); - Map configMap = mappingConfig.get(destination + "." + database + "." + table); + Map configMap = mappingConfig + .get(destination + "-" + groupId + "_" + database + "-" + table); if (configMap == null) { return false; @@ -169,7 +171,6 @@ public class RdbSyncService { return false; } - boolean executed = false; for (MappingConfig config : configs) { if (config.getConcurrent()) { List singleDmls = SingleDml.dml2SingleDmls(dml); @@ -186,9 +187,8 @@ public class RdbSyncService { dmlsPartition[hash].add(syncItem); }); } - executed = true; } - return executed; + return true; } }); } diff --git a/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml b/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml index ad0bb94b..74fb2fd0 100644 --- a/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml +++ b/client-adapter/rdb/src/main/resources/rdb/mytest_user.yml @@ -1,11 +1,12 @@ dataSourceKey: defaultDS destination: example -outerAdapterKey: oracle1 +groupId: g1 +outerAdapterKey: mysql1 concurrent: true dbMapping: database: mytest table: user - targetTable: mytest.tb_user + targetTable: mytest2.user targetPk: id: id # mapAll: true @@ -17,9 +18,10 @@ dbMapping: test1: -# Mirror schema synchronize config +## Mirror schema synchronize config #dataSourceKey: defaultDS #destination: example +#groupId: g1 #outerAdapterKey: mysql1 #concurrent: true #dbMapping: From 5284aa97931d05376e79e5bb917c28df9ea52312 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 1 Feb 2019 16:20:53 +0800 Subject: [PATCH 05/10] fix bugs --- .../adapter/config/YmlConfigBinder.java | 18 ++++++--- .../client/adapter/support/MessageUtil.java | 1 + .../canal/client/adapter/es/ESAdapter.java | 40 +++++++++++++++---- .../adapter/es/config/ESSyncConfigLoader.java | 4 +- .../client/adapter/hbase/HbaseAdapter.java | 29 ++++++++++---- .../hbase/config/MappingConfigLoader.java | 3 ++ .../canal/client/adapter/rdb/RdbAdapter.java | 18 ++++++--- .../adapter/rdb/config/ConfigLoader.java | 3 ++ .../rdb/service/RdbMirrorDbSyncService.java | 2 +- .../adapter/rdb/service/RdbSyncService.java | 29 +++++--------- 10 files changed, 101 insertions(+), 46 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java index 8d49e726..132199fc 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java @@ -67,11 +67,11 @@ public class YmlConfigBinder { } /** - * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: - * 当前内容有属性 zkServers: ${zookeeper.servers} - * 在envProperties中有属性 zookeeper.servers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 - * 则当前内容 zkServers 会被替换为 zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 - * 注: 假设绑定的类中 zkServers 属性是 List 对象, 则会自动映射成List + * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers} + * 在envProperties中有属性 zookeeper.servers: + * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为 + * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中 + * zkServers 属性是 List 对象, 则会自动映射成List * * @param prefix 指定前缀 * @param content yml内容 @@ -92,6 +92,10 @@ public class YmlConfigBinder { Resource configResource = new ByteArrayResource(contentBytes); PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); + if (propertySource == null) { + return null; + } + Properties properties = new Properties(); Map propertiesRes = new LinkedHashMap<>(); if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { @@ -126,6 +130,10 @@ public class YmlConfigBinder { propertiesRes.put(key, value); } + if (propertiesRes.isEmpty()) { + return null; + } + propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); T target = clazz.newInstance(); diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java index 239b8839..93ca987b 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java @@ -43,6 +43,7 @@ public class MessageUtil { dml.setTable(entry.getHeader().getTableName()); dml.setType(eventType.toString()); dml.setEs(entry.getHeader().getExecuteTime()); + dml.setIsDdl(rowChange.getIsDdl()); dml.setTs(System.currentTimeMillis()); dml.setSql(rowChange.getSql()); dmls.add(dml); diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index 9e54cce3..34b7a38a 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -1,7 +1,10 @@ package com.alibaba.otter.canal.client.adapter.es; import java.net.InetAddress; -import java.util.*; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -46,6 +49,8 @@ public class ESAdapter implements OuterAdapter { private ESConfigMonitor esConfigMonitor; + private Properties envProperties; + public TransportClient getTransportClient() { return transportClient; } @@ -65,6 +70,7 @@ public class ESAdapter implements OuterAdapter { @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { try { + this.envProperties = envProperties; Map esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); // 过滤不匹配的key的配置 esSyncConfigTmp.forEach((key, config) -> { @@ -93,10 +99,21 @@ public class ESAdapter implements OuterAdapter { String schema = matcher.group(2); schemaItem.getAliasTableItems().values().forEach(tableItem -> { - Map esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent( - StringUtils.trimToEmpty(config.getDestination()) + "-" + StringUtils - .trimToEmpty(config.getGroupId()) + "_" + schema + "-" + tableItem.getTableName(), - k -> new ConcurrentHashMap<>()); + Map esSyncConfigMap; + if (envProperties != null + && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + esSyncConfigMap = dbTableEsSyncConfig + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-" + + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-" + + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); + } else { + esSyncConfigMap = dbTableEsSyncConfig + .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-" + + tableItem.getTableName(), + k -> new ConcurrentHashMap<>()); + } + esSyncConfigMap.put(configName, config); }); } @@ -134,9 +151,16 @@ public class ESAdapter implements OuterAdapter { public void sync(Dml dml) { String database = dml.getDatabase(); String table = dml.getTable(); - Map configMap = dbTableEsSyncConfig - .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + "_" - + database + "-" + table); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId()) + + "_" + database + "-" + table); + } else { + configMap = dbTableEsSyncConfig + .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table); + } + if (configMap != null && !configMap.values().isEmpty()) { esSyncService.sync(configMap.values(), dml); } diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index d721cc3d..51216665 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -28,7 +28,9 @@ public class ESSyncConfigLoader { Map configContentMap = MappingConfigsLoader.loadConfigs("es"); configContentMap.forEach((fileName, content) -> { ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties); - + if (config == null) { + return; + } try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 795631e0..c307aec9 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -46,6 +46,8 @@ public class HbaseAdapter implements OuterAdapter { private HbaseConfigMonitor configMonitor; + private Properties envProperties; + public Map getHbaseMapping() { return hbaseMapping; } @@ -57,6 +59,7 @@ public class HbaseAdapter implements OuterAdapter { @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { try { + this.envProperties = envProperties; Map hbaseMappingTmp = MappingConfigLoader.load(envProperties); // 过滤不匹配的key的配置 hbaseMappingTmp.forEach((key, mappingConfig) -> { @@ -69,11 +72,19 @@ public class HbaseAdapter implements OuterAdapter { for (Map.Entry entry : hbaseMapping.entrySet()) { String configName = entry.getKey(); MappingConfig mappingConfig = entry.getValue(); - String k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" - + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" - + mappingConfig.getHbaseMapping().getDatabase() + "-" - + mappingConfig.getHbaseMapping().getTable(); - Map configMap = mappingConfigCache.computeIfAbsent(k, k1 -> new ConcurrentHashMap<>()); + String k; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + + mappingConfig.getHbaseMapping().getDatabase() + "-" + + mappingConfig.getHbaseMapping().getTable(); + } else { + k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + + mappingConfig.getHbaseMapping().getDatabase() + "-" + + mappingConfig.getHbaseMapping().getTable(); + } + Map configMap = mappingConfigCache.computeIfAbsent(k, + k1 -> new ConcurrentHashMap<>()); configMap.put(configName, mappingConfig); } @@ -108,8 +119,12 @@ public class HbaseAdapter implements OuterAdapter { String groupId = StringUtils.trimToEmpty(dml.getGroupId()); String database = dml.getDatabase(); String table = dml.getTable(); - Map configMap = mappingConfigCache - .get(destination + "-" + groupId + "_" + database + "-" + table); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table); + } else { + configMap = mappingConfigCache.get(destination + "_" + database + "-" + table); + } if (configMap != null) { List configs = new ArrayList<>(); configMap.values().forEach(config -> { diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index ff81e6b7..2b43492f 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -34,6 +34,9 @@ public class MappingConfigLoader { configContentMap.forEach((fileName, content) -> { MappingConfig config = YmlConfigBinder .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); + if (config == null) { + return; + } try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java index d06e03c1..8d1cab75 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java @@ -49,6 +49,8 @@ public class RdbAdapter implements OuterAdapter { private RdbConfigMonitor rdbConfigMonitor; + private Properties envProperties; + public Map getRdbMapping() { return rdbMapping; } @@ -68,6 +70,7 @@ public class RdbAdapter implements OuterAdapter { */ @Override public void init(OuterAdapterConfig configuration, Properties envProperties) { + this.envProperties = envProperties; Map rdbMappingTmp = ConfigLoader.load(envProperties); // 过滤不匹配的key的配置 rdbMappingTmp.forEach((key, mappingConfig) -> { @@ -86,10 +89,15 @@ public class RdbAdapter implements OuterAdapter { String configName = entry.getKey(); MappingConfig mappingConfig = entry.getValue(); if (!mappingConfig.getDbMapping().getMirrorDb()) { - String key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" - + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" - + mappingConfig.getDbMapping().getDatabase() + "-" - + mappingConfig.getDbMapping().getTable(); + String key; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable(); + } else { + key = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + + mappingConfig.getDbMapping().getDatabase() + "-" + mappingConfig.getDbMapping().getTable(); + } Map configMap = mappingConfigCache.computeIfAbsent(key, k1 -> new ConcurrentHashMap<>()); configMap.put(configName, mappingConfig); @@ -152,7 +160,7 @@ public class RdbAdapter implements OuterAdapter { return; } try { - rdbSyncService.sync(mappingConfigCache, dmls); + rdbSyncService.sync(mappingConfigCache, dmls, envProperties); rdbMirrorDbSyncService.sync(dmls); } catch (Exception e) { throw new RuntimeException(e); diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java index 63dd728a..514d9a79 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java @@ -36,6 +36,9 @@ public class ConfigLoader { configContentMap.forEach((fileName, content) -> { MappingConfig config = YmlConfigBinder .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); + if (config == null) { + return; + } try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java index eec96e0c..9d574fc6 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbMirrorDbSyncService.java @@ -59,7 +59,7 @@ public class RdbMirrorDbSyncService { if (mirrorDbConfig.getMappingConfig() == null) { continue; } - if (StringUtils.isNotEmpty(mirrorDbConfig.getMappingConfig().getGroupId())) { + if (dml.getGroupId() != null && StringUtils.isNotEmpty(mirrorDbConfig.getMappingConfig().getGroupId())) { if (!mirrorDbConfig.getMappingConfig().getGroupId().equals(dml.getGroupId())) { continue; // 如果groupId不匹配则过滤 } diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java index bf8d2de2..cc85b6b1 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java @@ -3,10 +3,7 @@ package com.alibaba.otter.canal.client.adapter.rdb.service; import java.sql.Connection; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.*; import java.util.function.Function; @@ -138,7 +135,7 @@ public class RdbSyncService { * @param mappingConfig 配置集合 * @param dmls 批量 DML */ - public void sync(Map> mappingConfig, List dmls) { + public void sync(Map> mappingConfig, List dmls, Properties envProperties) { sync(dmls, dml -> { if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { // DDL @@ -150,28 +147,22 @@ public class RdbSyncService { String groupId = StringUtils.trimToEmpty(dml.getGroupId()); String database = dml.getDatabase(); String table = dml.getTable(); - Map configMap = mappingConfig - .get(destination + "-" + groupId + "_" + database + "-" + table); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table); + } else { + configMap = mappingConfig.get(destination + "_" + database + "-" + table); + } if (configMap == null) { return false; } - List configs = new ArrayList<>(); - configMap.values().forEach(config -> { - if (StringUtils.isNotEmpty(config.getGroupId())) { - if (config.getGroupId().equals(dml.getGroupId())) { - configs.add(config); - } - } else { - configs.add(config); - } - }); - if (configs.isEmpty()) { + if (configMap.values().isEmpty()) { return false; } - for (MappingConfig config : configs) { + for (MappingConfig config : configMap.values()) { if (config.getConcurrent()) { List singleDmls = SingleDml.dml2SingleDmls(dml); singleDmls.forEach(singleDml -> { From f6d03bcb37033429abf765ab71f783a3c22e9da1 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 5 Feb 2019 17:48:08 +0800 Subject: [PATCH 06/10] fixed #1490 --- .../alibaba/otter/canal/client/adapter/es/ESAdapter.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index 34b7a38a..edd651f9 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -230,6 +230,9 @@ public class ESAdapter implements OuterAdapter { @Override public void destroy() { + if (esConfigMonitor != null) { + esConfigMonitor.destroy(); + } if (transportClient != null) { transportClient.close(); } @@ -237,10 +240,6 @@ public class ESAdapter implements OuterAdapter { @Override public String getDestination(String task) { - if (esConfigMonitor != null) { - esConfigMonitor.destroy(); - } - ESSyncConfig config = esSyncConfig.get(task); if (config != null) { return config.getDestination(); From ad62275cd73971f2925c21bf3be51accec1459b1 Mon Sep 17 00:00:00 2001 From: mcy Date: Tue, 5 Feb 2019 19:20:36 +0800 Subject: [PATCH 07/10] =?UTF-8?q?=E5=A2=9E=E5=8A=A0kafka=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89properties?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../otter/canal/deployer/CanalConstants.java | 1 + .../otter/canal/deployer/CanalStater.java | 12 +++++- deployer/src/main/resources/canal.properties | 3 +- .../otter/canal/common/MQProperties.java | 43 ++++++++++++------- .../otter/canal/kafka/CanalKafkaProducer.java | 7 ++- 5 files changed, 47 insertions(+), 19 deletions(-) diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java index 440d8091..f387e1b4 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java @@ -51,6 +51,7 @@ public class CanalConstants { public static final String CANAL_MQ_TRANSACTION = ROOT + "." + "mq.transaction"; public static final String CANAL_ALIYUN_ACCESSKEY = ROOT + "." + "aliyun.accessKey"; public static final String CANAL_ALIYUN_SECRETKEY = ROOT + "." + "aliyun.secretKey"; + public static final String CANAL_MQ_PROPERTIES = ROOT + "." + "mq.properties"; public static String getInstanceModeKey(String destination) { return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java index 19774812..05c8c6f0 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java @@ -1,5 +1,6 @@ package com.alibaba.otter.canal.deployer; +import java.util.Map; import java.util.Properties; import org.apache.commons.lang.StringUtils; @@ -162,7 +163,16 @@ public class CanalStater { if (!StringUtils.isEmpty(transaction)) { mqProperties.setTransaction(Boolean.valueOf(transaction)); } + + for (Object key : properties.keySet()) { + key = StringUtils.trim(key.toString()); + if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) { + String value = CanalController.getProperty(properties, (String) key); + String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1); + mqProperties.getProperties().put(subKey, value); + } + } + return mqProperties; } - } diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index fcdf6ac2..8b5c09fc 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -116,4 +116,5 @@ canal.mq.flatMessage = true canal.mq.compressionType = none canal.mq.acks = all # use transaction for kafka flatMessage batch produce -canal.mq.transaction = false \ No newline at end of file +canal.mq.transaction = false +#canal.mq.properties. = \ No newline at end of file diff --git a/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java b/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java index ccbf4f5e..4ee28c5e 100644 --- a/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java +++ b/server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java @@ -1,5 +1,7 @@ package com.alibaba.otter.canal.common; +import java.util.Properties; + /** * kafka 配置项 * @@ -8,22 +10,23 @@ package com.alibaba.otter.canal.common; */ public class MQProperties { - private String servers = "127.0.0.1:6667"; - private int retries = 0; - private int batchSize = 16384; - private int lingerMs = 1; - private int maxRequestSize = 1048576; - private long bufferMemory = 33554432L; - private boolean filterTransactionEntry = true; - private String producerGroup = "Canal-Producer"; - private int canalBatchSize = 50; - private Long canalGetTimeout = 100L; - private boolean flatMessage = true; - private String compressionType = "none"; - private String acks = "all"; - private String aliyunAccessKey = ""; - private String aliyunSecretKey = ""; - private boolean transaction = false; // 是否开启事务 + private String servers = "127.0.0.1:6667"; + private int retries = 0; + private int batchSize = 16384; + private int lingerMs = 1; + private int maxRequestSize = 1048576; + private long bufferMemory = 33554432L; + private boolean filterTransactionEntry = true; + private String producerGroup = "Canal-Producer"; + private int canalBatchSize = 50; + private Long canalGetTimeout = 100L; + private boolean flatMessage = true; + private String compressionType = "none"; + private String acks = "all"; + private String aliyunAccessKey = ""; + private String aliyunSecretKey = ""; + private boolean transaction = false; // 是否开启事务 + private Properties properties = new Properties(); public static class CanalDestination { @@ -210,4 +213,12 @@ public class MQProperties { public void setTransaction(boolean transaction) { this.transaction = transaction; } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } } diff --git a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java index 55133264..12c048aa 100644 --- a/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java +++ b/server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java @@ -49,7 +49,12 @@ public class CanalKafkaProducer implements CanalMQProducer { properties.put("max.request.size", kafkaProperties.getMaxRequestSize()); properties.put("buffer.memory", kafkaProperties.getBufferMemory()); properties.put("key.serializer", StringSerializer.class.getName()); - if(kafkaProperties.getTransaction()){ + + if (!kafkaProperties.getProperties().isEmpty()) { + properties.putAll(kafkaProperties.getProperties()); + } + + if (kafkaProperties.getTransaction()) { properties.put("transactional.id", "canal-transactional-id"); } else { properties.put("retries", kafkaProperties.getRetries()); From 0c5f5c128ff876a5dadbc22951b9c4f61ad8c74b Mon Sep 17 00:00:00 2001 From: rewer Date: Thu, 7 Feb 2019 13:49:35 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E6=95=B4=E7=90=86=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adapter/config/YmlConfigBinder.java | 312 ++--- .../bind/OriginCapablePropertyValue.java | 104 +- .../config/common/ByteArrayResource.java | 235 ++-- .../client/adapter/es/config/SchemaItem.java | 844 ++++++------- .../client/adapter/es/support/ESTemplate.java | 1052 ++++++++--------- .../hbase/service/HbaseEtlService.java | 789 +++++++------ .../launcher/loader/CanalAdapterLoader.java | 432 +++---- .../monitor/remote/DbRemoteConfigLoader.java | 494 ++++---- .../adapter/rdb/config/ConfigLoader.java | 104 +- .../adapter/rdb/monitor/RdbConfigMonitor.java | 339 +++--- .../adapter/rdb/service/RdbSyncService.java | 1005 ++++++++-------- .../client/kafka/KafkaCanalConnector.java | 651 +++++----- .../kafka/protocol/KafkaFlatMessage.java | 67 +- .../client/kafka/protocol/KafkaMessage.java | 68 +- 14 files changed, 3253 insertions(+), 3243 deletions(-) diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java index 132199fc..2cb0890d 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java @@ -1,156 +1,156 @@ -package com.alibaba.otter.canal.client.adapter.config; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; - -import org.springframework.util.PropertyPlaceholderHelper; -import org.springframework.util.StringUtils; - -import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; -import com.alibaba.otter.canal.client.adapter.config.common.*; - -/** - * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 - * - * @author reweerma 2019-2-1 上午9:14:02 - * @version 1.0.0 - */ -public class YmlConfigBinder { - - /** - * 将当前内容绑定到指定对象 - * - * @param content yml内容 - * @param clazz 指定对象类型 - * @return 对象 - */ - public static T bindYmlToObj(String content, Class clazz) { - return bindYmlToObj(null, content, clazz, null); - } - - /** - * 将当前内容绑定到指定对象并指定内容编码格式 - * - * @param content yml内容 - * @param clazz 指定对象类型 - * @param charset yml内容编码格式 - * @return 对象 - */ - public static T bindYmlToObj(String content, Class clazz, String charset) { - return bindYmlToObj(null, content, clazz, charset); - } - - /** - * 将当前内容指定前缀部分绑定到指定对象 - * - * @param prefix 指定前缀 - * @param content yml内容 - * @param clazz 指定对象类型 - * @return 对象 - */ - public static T bindYmlToObj(String prefix, String content, Class clazz) { - return bindYmlToObj(prefix, content, clazz, null); - } - - /** - * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 - * - * @param prefix 指定前缀 - * @param content yml内容 - * @param clazz 指定对象类型 - * @param charset yml内容编码格式 - * @return 对象 - */ - public static T bindYmlToObj(String prefix, String content, Class clazz, String charset) { - return bindYmlToObj(prefix, content, clazz, charset, null); - } - - /** - * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: ${zookeeper.servers} - * 在envProperties中有属性 zookeeper.servers: - * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为 - * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中 - * zkServers 属性是 List 对象, 则会自动映射成List - * - * @param prefix 指定前缀 - * @param content yml内容 - * @param clazz 指定对象类型 - * @param charset yml内容编码格式 - * @return 对象 - */ - public static T bindYmlToObj(String prefix, String content, Class clazz, String charset, - Properties baseProperties) { - try { - byte[] contentBytes; - if (charset == null) { - contentBytes = content.getBytes(); - } else { - contentBytes = content.getBytes(charset); - } - YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); - Resource configResource = new ByteArrayResource(contentBytes); - PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); - - if (propertySource == null) { - return null; - } - - Properties properties = new Properties(); - Map propertiesRes = new LinkedHashMap<>(); - if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { - prefix = prefix + "."; - } - - properties.putAll((Map) propertySource.getSource()); - - if (baseProperties != null) { - baseProperties.putAll(properties); - properties = baseProperties; - } - - for (Object o : ((Map) propertySource.getSource()).entrySet()) { - Map.Entry entry = (Map.Entry) o; - String key = (String) entry.getKey(); - Object value = entry.getValue(); - - if (prefix != null) { - if (key != null && key.startsWith(prefix)) { - key = key.substring(prefix.length()); - } else { - continue; - } - } - - if (value != null && value.toString().contains("${")) { - PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); - value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties); - } - - propertiesRes.put(key, value); - } - - if (propertiesRes.isEmpty()) { - return null; - } - - propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); - - T target = clazz.newInstance(); - - MutablePropertySources propertySources = new MutablePropertySources(); - propertySources.addFirst(propertySource); - - PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory(target); - factory.setPropertySources(propertySources); - factory.setIgnoreInvalidFields(true); - factory.setIgnoreUnknownFields(true); - - factory.bindPropertiesToTarget(); - - return target; - } catch (Exception e) { - throw new RuntimeException(e); - } - } -} +package com.alibaba.otter.canal.client.adapter.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.springframework.util.PropertyPlaceholderHelper; +import org.springframework.util.StringUtils; + +import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; +import com.alibaba.otter.canal.client.adapter.config.common.*; + +/** + * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 + * + * @author reweerma 2019-2-1 上午9:14:02 + * @version 1.0.0 + */ +public class YmlConfigBinder { + + /** + * 将当前内容绑定到指定对象 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz) { + return bindYmlToObj(null, content, clazz, null); + } + + /** + * 将当前内容绑定到指定对象并指定内容编码格式 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz, String charset) { + return bindYmlToObj(null, content, clazz, charset); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz) { + return bindYmlToObj(prefix, content, clazz, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset) { + return bindYmlToObj(prefix, content, clazz, charset, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: 当前内容有属性 zkServers: + * ${zookeeper.servers} 在envProperties中有属性 zookeeper.servers: + * 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 则当前内容 zkServers 会被替换为 + * zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 注: 假设绑定的类中 + * zkServers 属性是 List 对象, 则会自动映射成List + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset, + Properties baseProperties) { + try { + byte[] contentBytes; + if (charset == null) { + contentBytes = content.getBytes(); + } else { + contentBytes = content.getBytes(charset); + } + YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); + Resource configResource = new ByteArrayResource(contentBytes); + PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); + + if (propertySource == null) { + return null; + } + + Properties properties = new Properties(); + Map propertiesRes = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { + prefix = prefix + "."; + } + + properties.putAll((Map) propertySource.getSource()); + + if (baseProperties != null) { + baseProperties.putAll(properties); + properties = baseProperties; + } + + for (Object o : ((Map) propertySource.getSource()).entrySet()) { + Map.Entry entry = (Map.Entry) o; + String key = (String) entry.getKey(); + Object value = entry.getValue(); + + if (prefix != null) { + if (key != null && key.startsWith(prefix)) { + key = key.substring(prefix.length()); + } else { + continue; + } + } + + if (value != null && value.toString().contains("${")) { + PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); + value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties); + } + + propertiesRes.put(key, value); + } + + if (propertiesRes.isEmpty()) { + return null; + } + + propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); + + T target = clazz.newInstance(); + + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(propertySource); + + PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory(target); + factory.setPropertySources(propertySources); + factory.setIgnoreInvalidFields(true); + factory.setIgnoreUnknownFields(true); + + factory.bindPropertiesToTarget(); + + return target; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java index 184d3d0c..b79a03e8 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java @@ -1,52 +1,52 @@ -package com.alibaba.otter.canal.client.adapter.config.bind; - -import org.springframework.beans.PropertyValue; - -import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; - -/** - * A {@link PropertyValue} that can provide information about its origin. - * - * @author Andy Wilkinson - */ -class OriginCapablePropertyValue extends PropertyValue { - - private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; - - private final PropertyOrigin origin; - - OriginCapablePropertyValue(PropertyValue propertyValue){ - this(propertyValue.getName(), - propertyValue.getValue(), - (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); - } - - OriginCapablePropertyValue(String name, Object value, String originName, PropertySource originSource){ - this(name, value, new PropertyOrigin(originSource, originName)); - } - - OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ - super(name, value); - this.origin = origin; - setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); - } - - public PropertyOrigin getOrigin() { - return this.origin; - } - - @Override - public String toString() { - String name = (this.origin != null ? this.origin.getName() : this.getName()); - String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); - return "'" + name + "' from '" + source + "'"; - } - - public static PropertyOrigin getOrigin(PropertyValue propertyValue) { - if (propertyValue instanceof OriginCapablePropertyValue) { - return ((OriginCapablePropertyValue) propertyValue).getOrigin(); - } - return new OriginCapablePropertyValue(propertyValue).getOrigin(); - } - -} +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.beans.PropertyValue; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * A {@link PropertyValue} that can provide information about its origin. + * + * @author Andy Wilkinson + */ +class OriginCapablePropertyValue extends PropertyValue { + + private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; + + private final PropertyOrigin origin; + + OriginCapablePropertyValue(PropertyValue propertyValue){ + this(propertyValue.getName(), + propertyValue.getValue(), + (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); + } + + OriginCapablePropertyValue(String name, Object value, String originName, PropertySource originSource){ + this(name, value, new PropertyOrigin(originSource, originName)); + } + + OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ + super(name, value); + this.origin = origin; + setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); + } + + public PropertyOrigin getOrigin() { + return this.origin; + } + + @Override + public String toString() { + String name = (this.origin != null ? this.origin.getName() : this.getName()); + String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); + return "'" + name + "' from '" + source + "'"; + } + + public static PropertyOrigin getOrigin(PropertyValue propertyValue) { + if (propertyValue instanceof OriginCapablePropertyValue) { + return ((OriginCapablePropertyValue) propertyValue).getOrigin(); + } + return new OriginCapablePropertyValue(propertyValue).getOrigin(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java index 3ee85766..c8ec120b 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java @@ -1,118 +1,117 @@ -package com.alibaba.otter.canal.client.adapter.config.common; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; - -import org.springframework.core.io.InputStreamResource; -import org.springframework.core.io.InputStreamSource; -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; - -/** - * {@link Resource} implementation for a given byte array. - *

- * Creates a {@link ByteArrayInputStream} for the given byte array. - *

- * Useful for loading content from any given byte array, without having to - * resort to a single-use {@link InputStreamResource}. Particularly useful for - * creating mail attachments from local content, where JavaMail needs to be able - * to read the stream multiple times. - * - * @author Juergen Hoeller - * @author Sam Brannen - * @see ByteArrayInputStream - * @see InputStreamResource - * @since 1.2.3 - */ -public class ByteArrayResource extends AbstractResource { - - private final byte[] byteArray; - - private final String description; - - /** - * Create a new {@code ByteArrayResource}. - * - * @param byteArray the byte array to wrap - */ - public ByteArrayResource(byte[] byteArray){ - this(byteArray, "resource loaded from byte array"); - } - - /** - * Create a new {@code ByteArrayResource} with a description. - * - * @param byteArray the byte array to wrap - * @param description where the byte array comes from - */ - public ByteArrayResource(byte[] byteArray, String description){ - Assert.notNull(byteArray, "Byte array must not be null"); - this.byteArray = byteArray; - this.description = (description != null ? description : ""); - } - - /** - * Return the underlying byte array. - */ - public final byte[] getByteArray() { - return this.byteArray; - } - - /** - * This implementation always returns {@code true}. - */ - @Override - public boolean exists() { - return true; - } - - /** - * This implementation returns the length of the underlying byte array. - */ - @Override - public long contentLength() { - return this.byteArray.length; - } - - /** - * This implementation returns a ByteArrayInputStream for the underlying byte - * array. - * - * @see ByteArrayInputStream - */ - @Override - public InputStream getInputStream() throws IOException { - return new ByteArrayInputStream(this.byteArray); - } - - /** - * This implementation returns a description that includes the passed-in - * {@code description}, if any. - */ - @Override - public String getDescription() { - return "Byte array resource [" + this.description + "]"; - } - - /** - * This implementation compares the underlying byte array. - * - * @see Arrays#equals(byte[], byte[]) - */ - @Override - public boolean equals(Object obj) { - return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource - && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); - } - - /** - * This implementation returns the hash code based on the underlying byte array. - */ - @Override - public int hashCode() { - return (byte[].class.hashCode() * 29 * this.byteArray.length); - } - -} +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * {@link Resource} implementation for a given byte array. + *

+ * Creates a {@link ByteArrayInputStream} for the given byte array. + *

+ * Useful for loading content from any given byte array, without having to + * resort to a single-use {@link InputStreamResource}. Particularly useful for + * creating mail attachments from local content, where JavaMail needs to be able + * to read the stream multiple times. + * + * @author Juergen Hoeller + * @author Sam Brannen + * @see ByteArrayInputStream + * @see InputStreamResource + * @since 1.2.3 + */ +public class ByteArrayResource extends AbstractResource { + + private final byte[] byteArray; + + private final String description; + + /** + * Create a new {@code ByteArrayResource}. + * + * @param byteArray the byte array to wrap + */ + public ByteArrayResource(byte[] byteArray){ + this(byteArray, "resource loaded from byte array"); + } + + /** + * Create a new {@code ByteArrayResource} with a description. + * + * @param byteArray the byte array to wrap + * @param description where the byte array comes from + */ + public ByteArrayResource(byte[] byteArray, String description){ + Assert.notNull(byteArray, "Byte array must not be null"); + this.byteArray = byteArray; + this.description = (description != null ? description : ""); + } + + /** + * Return the underlying byte array. + */ + public final byte[] getByteArray() { + return this.byteArray; + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean exists() { + return true; + } + + /** + * This implementation returns the length of the underlying byte array. + */ + @Override + public long contentLength() { + return this.byteArray.length; + } + + /** + * This implementation returns a ByteArrayInputStream for the underlying byte + * array. + * + * @see ByteArrayInputStream + */ + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.byteArray); + } + + /** + * This implementation returns a description that includes the passed-in + * {@code description}, if any. + */ + @Override + public String getDescription() { + return "Byte array resource [" + this.description + "]"; + } + + /** + * This implementation compares the underlying byte array. + * + * @see Arrays#equals(byte[], byte[]) + */ + @Override + public boolean equals(Object obj) { + return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource + && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); + } + + /** + * This implementation returns the hash code based on the underlying byte array. + */ + @Override + public int hashCode() { + return (byte[].class.hashCode() * 29 * this.byteArray.length); + } + +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java index fc6e076c..5bb1eb41 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java @@ -1,422 +1,422 @@ -package com.alibaba.otter.canal.client.adapter.es.config; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; - -/** - * ES 映射配置视图 - * - * @author rewerma 2018-11-01 - * @version 1.0.0 - */ -public class SchemaItem { - - private Map aliasTableItems = new LinkedHashMap<>(); // 别名对应表名 - private Map selectFields = new LinkedHashMap<>(); // 查询字段 - private String sql; - - private volatile Map> tableItemAliases; - private volatile Map> columnFields; - private volatile Boolean allFieldsSimple; - - public void init() { - this.getTableItemAliases(); - this.getColumnFields(); - this.isAllFieldsSimple(); - aliasTableItems.values().forEach(tableItem -> { - tableItem.getRelationTableFields(); - tableItem.getRelationSelectFieldItems(); - }); - } - - public Map getAliasTableItems() { - return aliasTableItems; - } - - public void setAliasTableItems(Map aliasTableItems) { - this.aliasTableItems = aliasTableItems; - } - - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - public Map getSelectFields() { - return selectFields; - } - - public void setSelectFields(Map selectFields) { - this.selectFields = selectFields; - } - - public Map> getTableItemAliases() { - if (tableItemAliases == null) { - synchronized (SchemaItem.class) { - if (tableItemAliases == null) { - tableItemAliases = new LinkedHashMap<>(); - aliasTableItems.forEach((alias, tableItem) -> { - List aliases = tableItemAliases - .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>()); - aliases.add(tableItem); - }); - } - } - } - return tableItemAliases; - } - - public Map> getColumnFields() { - if (columnFields == null) { - synchronized (SchemaItem.class) { - if (columnFields == null) { - columnFields = new LinkedHashMap<>(); - getSelectFields() - .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { - TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); - // if (!tableItem.isSubQuery()) { - List fieldItems = columnFields.computeIfAbsent( - columnItem.getOwner() + "." + columnItem.getColumnName(), - k -> new ArrayList<>()); - fieldItems.add(fieldItem); - // } else { - // tableItem.getSubQueryFields().forEach(subQueryField -> { - // List fieldItems = columnFields.computeIfAbsent( - // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), - // k -> new ArrayList<>()); - // fieldItems.add(fieldItem); - // }); - // } - })); - } - } - } - return columnFields; - } - - public boolean isAllFieldsSimple() { - if (allFieldsSimple == null) { - synchronized (SchemaItem.class) { - if (allFieldsSimple == null) { - allFieldsSimple = true; - - for (FieldItem fieldItem : getSelectFields().values()) { - if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { - allFieldsSimple = false; - break; - } - } - } - } - } - - return allFieldsSimple; - } - - public TableItem getMainTable() { - if (!aliasTableItems.isEmpty()) { - return aliasTableItems.values().iterator().next(); - } else { - return null; - } - } - - public FieldItem getIdFieldItem(ESMapping mapping) { - if (mapping.get_id() != null) { - return getSelectFields().get(mapping.get_id()); - } else { - return getSelectFields().get(mapping.getPk()); - } - } - - public static class TableItem { - - private SchemaItem schemaItem; - - private String schema; - private String tableName; - private String alias; - private String subQuerySql; - private List subQueryFields = new ArrayList<>(); - private List relationFields = new ArrayList<>(); - - private boolean main; - private boolean subQuery; - - private volatile Map> relationTableFields; // 当前表关联条件字段对应主表查询字段 - private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 - - public TableItem(SchemaItem schemaItem){ - this.schemaItem = schemaItem; - } - - public SchemaItem getSchemaItem() { - return schemaItem; - } - - public void setSchemaItem(SchemaItem schemaItem) { - this.schemaItem = schemaItem; - } - - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getAlias() { - return alias; - } - - public void setAlias(String alias) { - this.alias = alias; - } - - public String getSubQuerySql() { - return subQuerySql; - } - - public void setSubQuerySql(String subQuerySql) { - this.subQuerySql = subQuerySql; - } - - public boolean isMain() { - return main; - } - - public void setMain(boolean main) { - this.main = main; - } - - public boolean isSubQuery() { - return subQuery; - } - - public void setSubQuery(boolean subQuery) { - this.subQuery = subQuery; - } - - public List getSubQueryFields() { - return subQueryFields; - } - - public void setSubQueryFields(List subQueryFields) { - this.subQueryFields = subQueryFields; - } - - public List getRelationFields() { - return relationFields; - } - - public void setRelationFields(List relationFields) { - this.relationFields = relationFields; - } - - public Map> getRelationTableFields() { - if (relationTableFields == null) { - synchronized (SchemaItem.class) { - if (relationTableFields == null) { - relationTableFields = new LinkedHashMap<>(); - - getRelationFields().forEach(relationFieldsPair -> { - FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); - FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); - FieldItem currentTableRelField = null; - if (getAlias().equals(leftFieldItem.getOwner())) { - currentTableRelField = leftFieldItem; - } else if (getAlias().equals(rightFieldItem.getOwner())) { - currentTableRelField = rightFieldItem; - } - - if (currentTableRelField != null) { - List selectFieldItem = getSchemaItem().getColumnFields() - .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); - if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - relationTableFields.put(currentTableRelField, selectFieldItem); - } else { - selectFieldItem = getSchemaItem().getColumnFields() - .get(rightFieldItem.getOwner() + "." - + rightFieldItem.getColumn().getColumnName()); - if (selectFieldItem != null && !selectFieldItem.isEmpty()) { - relationTableFields.put(currentTableRelField, selectFieldItem); - } else { - throw new UnsupportedOperationException( - "Relation condition column must in select columns."); - } - } - } - }); - } - } - } - return relationTableFields; - } - - public List getRelationSelectFieldItems() { - if (relationSelectFieldItems == null) { - synchronized (SchemaItem.class) { - if (relationSelectFieldItems == null) { - relationSelectFieldItems = new ArrayList<>(); - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - if (fieldItem.getOwners().contains(getAlias())) { - relationSelectFieldItems.add(fieldItem); - } - } - } - } - } - return relationSelectFieldItems; - } - } - - public static class RelationFieldsPair { - - private FieldItem leftFieldItem; - private FieldItem rightFieldItem; - - public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){ - this.leftFieldItem = leftFieldItem; - this.rightFieldItem = rightFieldItem; - } - - public FieldItem getLeftFieldItem() { - return leftFieldItem; - } - - public void setLeftFieldItem(FieldItem leftFieldItem) { - this.leftFieldItem = leftFieldItem; - } - - public FieldItem getRightFieldItem() { - return rightFieldItem; - } - - public void setRightFieldItem(FieldItem rightFieldItem) { - this.rightFieldItem = rightFieldItem; - } - } - - public static class FieldItem { - - private String fieldName; - private List columnItems = new ArrayList<>(); - private List owners = new ArrayList<>(); - - private boolean method; - private boolean binaryOp; - - public String getFieldName() { - return fieldName; - } - - public void setFieldName(String fieldName) { - this.fieldName = fieldName; - } - - public List getColumnItems() { - return columnItems; - } - - public void setColumnItems(List columnItems) { - this.columnItems = columnItems; - } - - public boolean isMethod() { - return method; - } - - public void setMethod(boolean method) { - this.method = method; - } - - public boolean isBinaryOp() { - return binaryOp; - } - - public void setBinaryOp(boolean binaryOp) { - this.binaryOp = binaryOp; - } - - public List getOwners() { - return owners; - } - - public void setOwners(List owners) { - this.owners = owners; - } - - public void addColumn(ColumnItem columnItem) { - columnItems.add(columnItem); - } - - public ColumnItem getColumn() { - if (!columnItems.isEmpty()) { - return columnItems.get(0); - } else { - return null; - } - } - - public String getOwner() { - if (!owners.isEmpty()) { - return owners.get(0); - } else { - return null; - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - FieldItem fieldItem = (FieldItem) o; - - return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null; - } - - @Override - public int hashCode() { - return fieldName != null ? fieldName.hashCode() : 0; - } - } - - public static class ColumnItem { - - private String owner; - private String columnName; - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getColumnName() { - return columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - } -} +package com.alibaba.otter.canal.client.adapter.es.config; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; + +/** + * ES 映射配置视图 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class SchemaItem { + + private Map aliasTableItems = new LinkedHashMap<>(); // 别名对应表名 + private Map selectFields = new LinkedHashMap<>(); // 查询字段 + private String sql; + + private volatile Map> tableItemAliases; + private volatile Map> columnFields; + private volatile Boolean allFieldsSimple; + + public void init() { + this.getTableItemAliases(); + this.getColumnFields(); + this.isAllFieldsSimple(); + aliasTableItems.values().forEach(tableItem -> { + tableItem.getRelationTableFields(); + tableItem.getRelationSelectFieldItems(); + }); + } + + public Map getAliasTableItems() { + return aliasTableItems; + } + + public void setAliasTableItems(Map aliasTableItems) { + this.aliasTableItems = aliasTableItems; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Map getSelectFields() { + return selectFields; + } + + public void setSelectFields(Map selectFields) { + this.selectFields = selectFields; + } + + public Map> getTableItemAliases() { + if (tableItemAliases == null) { + synchronized (SchemaItem.class) { + if (tableItemAliases == null) { + tableItemAliases = new LinkedHashMap<>(); + aliasTableItems.forEach((alias, tableItem) -> { + List aliases = tableItemAliases + .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>()); + aliases.add(tableItem); + }); + } + } + } + return tableItemAliases; + } + + public Map> getColumnFields() { + if (columnFields == null) { + synchronized (SchemaItem.class) { + if (columnFields == null) { + columnFields = new LinkedHashMap<>(); + getSelectFields() + .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> { + // TableItem tableItem = getAliasTableItems().get(columnItem.getOwner()); + // if (!tableItem.isSubQuery()) { + List fieldItems = columnFields.computeIfAbsent( + columnItem.getOwner() + "." + columnItem.getColumnName(), + k -> new ArrayList<>()); + fieldItems.add(fieldItem); + // } else { + // tableItem.getSubQueryFields().forEach(subQueryField -> { + // List fieldItems = columnFields.computeIfAbsent( + // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(), + // k -> new ArrayList<>()); + // fieldItems.add(fieldItem); + // }); + // } + })); + } + } + } + return columnFields; + } + + public boolean isAllFieldsSimple() { + if (allFieldsSimple == null) { + synchronized (SchemaItem.class) { + if (allFieldsSimple == null) { + allFieldsSimple = true; + + for (FieldItem fieldItem : getSelectFields().values()) { + if (fieldItem.isMethod() || fieldItem.isBinaryOp()) { + allFieldsSimple = false; + break; + } + } + } + } + } + + return allFieldsSimple; + } + + public TableItem getMainTable() { + if (!aliasTableItems.isEmpty()) { + return aliasTableItems.values().iterator().next(); + } else { + return null; + } + } + + public FieldItem getIdFieldItem(ESMapping mapping) { + if (mapping.get_id() != null) { + return getSelectFields().get(mapping.get_id()); + } else { + return getSelectFields().get(mapping.getPk()); + } + } + + public static class TableItem { + + private SchemaItem schemaItem; + + private String schema; + private String tableName; + private String alias; + private String subQuerySql; + private List subQueryFields = new ArrayList<>(); + private List relationFields = new ArrayList<>(); + + private boolean main; + private boolean subQuery; + + private volatile Map> relationTableFields; // 当前表关联条件字段对应主表查询字段 + private volatile List relationSelectFieldItems; // 子表所在主表的查询字段 + + public TableItem(SchemaItem schemaItem){ + this.schemaItem = schemaItem; + } + + public SchemaItem getSchemaItem() { + return schemaItem; + } + + public void setSchemaItem(SchemaItem schemaItem) { + this.schemaItem = schemaItem; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getSubQuerySql() { + return subQuerySql; + } + + public void setSubQuerySql(String subQuerySql) { + this.subQuerySql = subQuerySql; + } + + public boolean isMain() { + return main; + } + + public void setMain(boolean main) { + this.main = main; + } + + public boolean isSubQuery() { + return subQuery; + } + + public void setSubQuery(boolean subQuery) { + this.subQuery = subQuery; + } + + public List getSubQueryFields() { + return subQueryFields; + } + + public void setSubQueryFields(List subQueryFields) { + this.subQueryFields = subQueryFields; + } + + public List getRelationFields() { + return relationFields; + } + + public void setRelationFields(List relationFields) { + this.relationFields = relationFields; + } + + public Map> getRelationTableFields() { + if (relationTableFields == null) { + synchronized (SchemaItem.class) { + if (relationTableFields == null) { + relationTableFields = new LinkedHashMap<>(); + + getRelationFields().forEach(relationFieldsPair -> { + FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem(); + FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem(); + FieldItem currentTableRelField = null; + if (getAlias().equals(leftFieldItem.getOwner())) { + currentTableRelField = leftFieldItem; + } else if (getAlias().equals(rightFieldItem.getOwner())) { + currentTableRelField = rightFieldItem; + } + + if (currentTableRelField != null) { + List selectFieldItem = getSchemaItem().getColumnFields() + .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + selectFieldItem = getSchemaItem().getColumnFields() + .get(rightFieldItem.getOwner() + "." + + rightFieldItem.getColumn().getColumnName()); + if (selectFieldItem != null && !selectFieldItem.isEmpty()) { + relationTableFields.put(currentTableRelField, selectFieldItem); + } else { + throw new UnsupportedOperationException( + "Relation condition column must in select columns."); + } + } + } + }); + } + } + } + return relationTableFields; + } + + public List getRelationSelectFieldItems() { + if (relationSelectFieldItems == null) { + synchronized (SchemaItem.class) { + if (relationSelectFieldItems == null) { + relationSelectFieldItems = new ArrayList<>(); + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getOwners().contains(getAlias())) { + relationSelectFieldItems.add(fieldItem); + } + } + } + } + } + return relationSelectFieldItems; + } + } + + public static class RelationFieldsPair { + + private FieldItem leftFieldItem; + private FieldItem rightFieldItem; + + public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){ + this.leftFieldItem = leftFieldItem; + this.rightFieldItem = rightFieldItem; + } + + public FieldItem getLeftFieldItem() { + return leftFieldItem; + } + + public void setLeftFieldItem(FieldItem leftFieldItem) { + this.leftFieldItem = leftFieldItem; + } + + public FieldItem getRightFieldItem() { + return rightFieldItem; + } + + public void setRightFieldItem(FieldItem rightFieldItem) { + this.rightFieldItem = rightFieldItem; + } + } + + public static class FieldItem { + + private String fieldName; + private List columnItems = new ArrayList<>(); + private List owners = new ArrayList<>(); + + private boolean method; + private boolean binaryOp; + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public List getColumnItems() { + return columnItems; + } + + public void setColumnItems(List columnItems) { + this.columnItems = columnItems; + } + + public boolean isMethod() { + return method; + } + + public void setMethod(boolean method) { + this.method = method; + } + + public boolean isBinaryOp() { + return binaryOp; + } + + public void setBinaryOp(boolean binaryOp) { + this.binaryOp = binaryOp; + } + + public List getOwners() { + return owners; + } + + public void setOwners(List owners) { + this.owners = owners; + } + + public void addColumn(ColumnItem columnItem) { + columnItems.add(columnItem); + } + + public ColumnItem getColumn() { + if (!columnItems.isEmpty()) { + return columnItems.get(0); + } else { + return null; + } + } + + public String getOwner() { + if (!owners.isEmpty()) { + return owners.get(0); + } else { + return null; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + FieldItem fieldItem = (FieldItem) o; + + return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null; + } + + @Override + public int hashCode() { + return fieldName != null ? fieldName.hashCode() : 0; + } + } + + public static class ColumnItem { + + private String owner; + private String columnName; + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + } +} diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index e57b89b7..9be2d801 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -1,526 +1,526 @@ -package com.alibaba.otter.canal.client.adapter.es.support; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -import javax.sql.DataSource; - -import com.alibaba.fastjson.JSON; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.UpdateByQueryAction; -import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.SearchHit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.CollectionUtils; - -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; - -/** - * ES 操作模板 - * - * @author rewerma 2018-11-01 - * @version 1.0.0 - */ -public class ESTemplate { - - private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); - - private static final int MAX_BATCH_SIZE = 1000; - - private TransportClient transportClient; - - public ESTemplate(TransportClient transportClient){ - this.transportClient = transportClient; - } - - /** - * 插入数据 - * - * @param mapping - * @param pkVal - * @param esFieldData - * @return - */ - public boolean insert(ESMapping mapping, Object pkVal, Map esFieldData) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - if (mapping.get_id() != null) { - bulkRequestBuilder - .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) - .setSource(esFieldData)); - } else { - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); - for (SearchHit hit : response.getHits()) { - bulkRequestBuilder - .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); - } - bulkRequestBuilder - .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData)); - } - return commitBulkRequest(bulkRequestBuilder); - } - - /** - * 根据主键更新数据 - * - * @param mapping - * @param pkVal - * @param esFieldData - * @return - */ - public boolean update(ESMapping mapping, Object pkVal, Map esFieldData) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData); - return commitBulkRequest(bulkRequestBuilder); - } - - public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, - Map esFieldData) { - if (mapping.get_id() != null) { - bulkRequestBuilder - .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) - .setDoc(esFieldData)); - } else { - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); - for (SearchHit hit : response.getHits()) { - bulkRequestBuilder - .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) - .setDoc(esFieldData)); - } - } - } - - /** - * update by query - * - * @param config - * @param paramsTmp - * @param esFieldData - * @return - */ - public boolean updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { - if (paramsTmp.isEmpty()) { - return false; - } - ESMapping mapping = config.getEsMapping(); - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); - - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setSize(0) - .setQuery(queryBuilder) - .get(); - long count = response.getHits().getTotalHits(); - // 如果更新量大于Max, 查询sql批量更新 - if (count > MAX_BATCH_SIZE) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - - DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); - // 查询sql更新 - StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); - paramsTmp.forEach( - (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND ")); - int len = sql.length(); - sql.delete(len - 4, len); - ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { - int exeCount = 1; - try { - BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder; - while (rs.next()) { - Object idVal = getIdValFromRS(mapping, rs); - append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); - - if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { - commitBulkRequest(bulkRequestBuilderTmp); - bulkRequestBuilderTmp = transportClient.prepareBulk(); - } - exeCount++; - } - - if (bulkRequestBuilder.numberOfActions() > 0) { - commitBulkRequest(bulkRequestBuilderTmp); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - return 0; - }); - return true; - } else { - return updateByQuery(mapping, queryBuilder, esFieldData, 1); - } - } - - private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, - int counter) { - if (CollectionUtils.isEmpty(esFieldData)) { - return true; - } - - StringBuilder sb = new StringBuilder(); - esFieldData.forEach((key, value) -> { - if (value instanceof Map) { - HashMap mapValue = (HashMap) value; - if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) { - sb.append("ctx._source") - .append("['") - .append(key) - .append("']") - .append(" = [") - .append(mapValue.get("lon")) - .append(", ") - .append(mapValue.get("lat")) - .append("];"); - } else { - sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); - sb.append(JSON.toJSONString(value)); - sb.append(";"); - } - } else if (value instanceof List) { - sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); - sb.append(JSON.toJSONString(value)); - sb.append(";"); - } else if (value instanceof String) { - sb.append("ctx._source") - .append("['") - .append(key) - .append("']") - .append(" = '") - .append(value) - .append("';"); - } else { - sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";"); - } - }); - String scriptLine = sb.toString(); - if (logger.isTraceEnabled()) { - logger.trace(scriptLine); - } - - UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); - updateByQuery.source(mapping.get_index()) - .abortOnVersionConflict(false) - .filter(queryBuilder) - .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); - - BulkByScrollResponse response = updateByQuery.get(); - if (logger.isTraceEnabled()) { - logger.trace("updateByQuery response: {}", response.getStatus()); - } - if (!CollectionUtils.isEmpty(response.getSearchFailures())) { - logger.error("script update_for_search has search error: " + response.getBulkFailures()); - return false; - } - - if (!CollectionUtils.isEmpty(response.getBulkFailures())) { - logger.error("script update_for_search has update error: " + response.getBulkFailures()); - return false; - } - - if (response.getStatus().getVersionConflicts() > 0) { - if (counter >= 3) { - logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter); - return false; - } - logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行..."); - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - // ignore - } - return updateByQuery(mapping, queryBuilder, esFieldData, ++counter); - } - - return true; - } - - /** - * 通过主键删除数据 - * - * @param mapping - * @param pkVal - * @return - */ - public boolean delete(ESMapping mapping, Object pkVal) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - if (mapping.get_id() != null) { - bulkRequestBuilder - .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); - } else { - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); - for (SearchHit hit : response.getHits()) { - bulkRequestBuilder - .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); - } - } - return commitBulkRequest(bulkRequestBuilder); - } - - /** - * 批量提交 - * - * @param bulkRequestBuilder - * @return - */ - private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { - if (bulkRequestBuilder.numberOfActions() > 0) { - BulkResponse response = bulkRequestBuilder.execute().actionGet(); - if (response.hasFailures()) { - for (BulkItemResponse itemResponse : response.getItems()) { - if (!itemResponse.isFailed()) { - continue; - } - - if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { - logger.warn(itemResponse.getFailureMessage()); - } else { - logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); - } - } - } - - return !response.hasFailures(); - } - return true; - } - - public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, - String columnName) throws SQLException { - String esType = getEsType(mapping, fieldName); - - Object value = resultSet.getObject(columnName); - if (value instanceof Boolean) { - if (!"boolean".equals(esType)) { - value = resultSet.getByte(columnName); - } - } - - // 如果是对象类型 - if (mapping.getObjFields().containsKey(fieldName)) { - return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); - } else { - return ESSyncUtil.typeConvert(value, esType); - } - } - - public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, - Map esFieldData) throws SQLException { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = value; - } - - if (!fieldItem.getFieldName().equals(mapping.get_id()) - && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), value); - } - } - return resultIdVal; - } - - public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = value; - break; - } - } - return resultIdVal; - } - - public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, - Map esFieldData) throws SQLException { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); - } - - for (ColumnItem columnItem : fieldItem.getColumnItems()) { - if (dmlOld.containsKey(columnItem.getColumnName()) - && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), - getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); - break; - } - } - } - return resultIdVal; - } - - public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { - String esType = getEsType(mapping, fieldName); - Object value = dmlData.get(columnName); - if (value instanceof Byte) { - if ("boolean".equals(esType)) { - value = ((Byte) value).intValue() != 0; - } - } - - // 如果是对象类型 - if (mapping.getObjFields().containsKey(fieldName)) { - return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); - } else { - return ESSyncUtil.typeConvert(value, esType); - } - } - - /** - * 将dml的data转换为es的data - * - * @param mapping 配置mapping - * @param dmlData dml data - * @param esFieldData es data - * @return 返回 id 值 - */ - public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, - Map esFieldData) { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); - Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = value; - } - - if (!fieldItem.getFieldName().equals(mapping.get_id()) - && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), value); - } - } - return resultIdVal; - } - - /** - * 将dml的data, old转换为es的data - * - * @param mapping 配置mapping - * @param dmlData dml data - * @param esFieldData es data - * @return 返回 id 值 - */ - public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, - Map esFieldData) { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); - } - - if (dmlOld.get(columnName) != null && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), - getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); - } - } - return resultIdVal; - } - - /** - * es 字段类型本地缓存 - */ - private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); - - /** - * 获取es mapping中的属性类型 - * - * @param mapping mapping配置 - * @param fieldName 属性名 - * @return 类型 - */ - @SuppressWarnings("unchecked") - private String getEsType(ESMapping mapping, String fieldName) { - String key = mapping.get_index() + "-" + mapping.get_type(); - Map fieldType = esFieldTypes.get(key); - if (fieldType == null) { - ImmutableOpenMap mappings; - try { - mappings = transportClient.admin() - .cluster() - .prepareState() - .execute() - .actionGet() - .getState() - .getMetaData() - .getIndices() - .get(mapping.get_index()) - .getMappings(); - } catch (NullPointerException e) { - throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); - } - MappingMetaData mappingMetaData = mappings.get(mapping.get_type()); - if (mappingMetaData == null) { - throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); - } - - fieldType = new LinkedHashMap<>(); - - Map sourceMap = mappingMetaData.getSourceAsMap(); - Map esMapping = (Map) sourceMap.get("properties"); - for (Map.Entry entry : esMapping.entrySet()) { - Map value = (Map) entry.getValue(); - if (value.containsKey("properties")) { - fieldType.put(entry.getKey(), "object"); - } else { - fieldType.put(entry.getKey(), (String) value.get("type")); - } - } - esFieldTypes.put(key, fieldType); - } - - return fieldType.get(fieldName); - } -} +package com.alibaba.otter.canal.client.adapter.es.support; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import com.alibaba.fastjson.JSON; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.UpdateByQueryAction; +import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +/** + * ES 操作模板 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESTemplate { + + private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); + + private static final int MAX_BATCH_SIZE = 1000; + + private TransportClient transportClient; + + public ESTemplate(TransportClient transportClient){ + this.transportClient = transportClient; + } + + /** + * 插入数据 + * + * @param mapping + * @param pkVal + * @param esFieldData + * @return + */ + public boolean insert(ESMapping mapping, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setSource(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData)); + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 根据主键更新数据 + * + * @param mapping + * @param pkVal + * @param esFieldData + * @return + */ + public boolean update(ESMapping mapping, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData); + return commitBulkRequest(bulkRequestBuilder); + } + + public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, + Map esFieldData) { + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setDoc(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) + .setDoc(esFieldData)); + } + } + } + + /** + * update by query + * + * @param config + * @param paramsTmp + * @param esFieldData + * @return + */ + public boolean updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { + if (paramsTmp.isEmpty()) { + return false; + } + ESMapping mapping = config.getEsMapping(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); + + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setSize(0) + .setQuery(queryBuilder) + .get(); + long count = response.getHits().getTotalHits(); + // 如果更新量大于Max, 查询sql批量更新 + if (count > MAX_BATCH_SIZE) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + // 查询sql更新 + StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); + paramsTmp.forEach( + (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND ")); + int len = sql.length(); + sql.delete(len - 4, len); + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + int exeCount = 1; + try { + BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder; + while (rs.next()) { + Object idVal = getIdValFromRS(mapping, rs); + append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); + + if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + bulkRequestBuilderTmp = transportClient.prepareBulk(); + } + exeCount++; + } + + if (bulkRequestBuilder.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + return true; + } else { + return updateByQuery(mapping, queryBuilder, esFieldData, 1); + } + } + + private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, + int counter) { + if (CollectionUtils.isEmpty(esFieldData)) { + return true; + } + + StringBuilder sb = new StringBuilder(); + esFieldData.forEach((key, value) -> { + if (value instanceof Map) { + Map mapValue = (Map) value; + if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = [") + .append(mapValue.get("lon")) + .append(", ") + .append(mapValue.get("lat")) + .append("];"); + } else { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); + } + } else if (value instanceof List) { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); + } else if (value instanceof String) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = '") + .append(value) + .append("';"); + } else { + sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";"); + } + }); + String scriptLine = sb.toString(); + if (logger.isTraceEnabled()) { + logger.trace(scriptLine); + } + + UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); + updateByQuery.source(mapping.get_index()) + .abortOnVersionConflict(false) + .filter(queryBuilder) + .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); + + BulkByScrollResponse response = updateByQuery.get(); + if (logger.isTraceEnabled()) { + logger.trace("updateByQuery response: {}", response.getStatus()); + } + if (!CollectionUtils.isEmpty(response.getSearchFailures())) { + logger.error("script update_for_search has search error: " + response.getBulkFailures()); + return false; + } + + if (!CollectionUtils.isEmpty(response.getBulkFailures())) { + logger.error("script update_for_search has update error: " + response.getBulkFailures()); + return false; + } + + if (response.getStatus().getVersionConflicts() > 0) { + if (counter >= 3) { + logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter); + return false; + } + logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + // ignore + } + return updateByQuery(mapping, queryBuilder, esFieldData, ++counter); + } + + return true; + } + + /** + * 通过主键删除数据 + * + * @param mapping + * @param pkVal + * @return + */ + public boolean delete(ESMapping mapping, Object pkVal) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 批量提交 + * + * @param bulkRequestBuilder + * @return + */ + private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { + if (bulkRequestBuilder.numberOfActions() > 0) { + BulkResponse response = bulkRequestBuilder.execute().actionGet(); + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.warn(itemResponse.getFailureMessage()); + } else { + logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); + } + } + } + + return !response.hasFailures(); + } + return true; + } + + public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException { + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(columnName); + if (value instanceof Boolean) { + if (!"boolean".equals(esType)) { + value = resultSet.getByte(columnName); + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); + } + } + return resultIdVal; + } + + public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + break; + } + } + return resultIdVal; + } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + } + + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (dmlOld.containsKey(columnItem.getColumnName()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); + break; + } + } + } + return resultIdVal; + } + + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(columnName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + /** + * 将dml的data转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); + } + } + return resultIdVal; + } + + /** + * 将dml的data, old转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + } + + if (dmlOld.get(columnName) != null && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); + } + } + return resultIdVal; + } + + /** + * es 字段类型本地缓存 + */ + private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + + /** + * 获取es mapping中的属性类型 + * + * @param mapping mapping配置 + * @param fieldName 属性名 + * @return 类型 + */ + @SuppressWarnings("unchecked") + private String getEsType(ESMapping mapping, String fieldName) { + String key = mapping.get_index() + "-" + mapping.get_type(); + Map fieldType = esFieldTypes.get(key); + if (fieldType == null) { + ImmutableOpenMap mappings; + try { + mappings = transportClient.admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices() + .get(mapping.get_index()) + .getMappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + MappingMetaData mappingMetaData = mappings.get(mapping.get_type()); + if (mappingMetaData == null) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + + fieldType = new LinkedHashMap<>(); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); + if (value.containsKey("properties")) { + fieldType.put(entry.getKey(), "object"); + } else { + fieldType.put(entry.getKey(), (String) value.get("type")); + } + } + esFieldTypes.put(key, fieldType); + } + + return fieldType.get(fieldName); + } +} diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java index 172592e2..d5a2ef58 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseEtlService.java @@ -1,392 +1,397 @@ -package com.alibaba.otter.canal.client.adapter.hbase.service; - -import java.sql.*; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; - -import javax.sql.DataSource; - -import com.alibaba.otter.canal.client.adapter.support.Util; -import org.apache.hadoop.hbase.util.Bytes; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; -import com.alibaba.otter.canal.client.adapter.hbase.support.*; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; -import com.google.common.base.Joiner; - -/** - * HBase ETL 操作业务类 - * - * @author rewerma @ 2018-10-20 - * @version 1.0.0 - */ -public class HbaseEtlService { - - private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); - - - /** - * 建表 - * - * @param hbaseTemplate - * @param config - */ - public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { - try { - // 判断hbase表是否存在，不存在则建表 - MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); - if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { - hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily()); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException(e); - } - } - - /** - * 导入数据 - * - * @param ds 数据源 - * @param hbaseTemplate hbaseTemplate - * @param config 配置 - * @param params 筛选条件 - * @return 导入结果 - */ - public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, - List params) { - EtlResult etlResult = new EtlResult(); - AtomicLong successCount = new AtomicLong(); - List errMsg = new ArrayList<>(); - String hbaseTable = ""; - try { - if (config == null) { - logger.error("Config is null!"); - etlResult.setSucceeded(false); - etlResult.setErrorMessage("Config is null!"); - return etlResult; - } - MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); - hbaseTable = hbaseMapping.getHbaseTable(); - - long start = System.currentTimeMillis(); - - if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { - logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!"); - // 如果表存在则删除 - if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { - hbaseTemplate.disableTable(hbaseMapping.getHbaseTable()); - hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable()); - } - params = null; - } else { - logger.info(hbaseMapping.getHbaseTable() + " etl is starting!"); - } - createTable(hbaseTemplate, config); - - // 拼接sql - String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable(); - - // 拼接条件 - if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) { - AtomicBoolean stExists = new AtomicBoolean(false); - // 验证是否有SYS_TIME字段 - Util.sqlRS(ds, sql, rs -> { - try { - ResultSetMetaData rsmd = rs.getMetaData(); - int cnt = rsmd.getColumnCount(); - for (int i = 1; i <= cnt; i++) { - String columnName = rsmd.getColumnName(i); - if ("SYS_TIME".equalsIgnoreCase(columnName)) { - stExists.set(true); - break; - } - } - } catch (Exception e) { - // ignore - } - return null; - }); - if (stExists.get()) { - sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; - } - } else if (hbaseMapping.getEtlCondition() != null && params != null) { - String etlCondition = hbaseMapping.getEtlCondition(); - int size = params.size(); - for (int i = 0; i < size; i++) { - etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); - } - - sql += " " + etlCondition; - } - - // 获取总数 - String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; - long cnt = (Long) Util.sqlRS(ds, countSql, rs -> { - Long count = null; - try { - if (rs.next()) { - count = ((Number) rs.getObject(1)).longValue(); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - return count == null ? 0 : count; - }); - - // 当大于1万条记录时开启多线程 - if (cnt >= 10000) { - int threadCount = 3; - long perThreadCnt = cnt / threadCount; - ExecutorService executor = Executors.newFixedThreadPool(threadCount); - List> futures = new ArrayList<>(threadCount); - for (int i = 0; i < threadCount; i++) { - long offset = i * perThreadCnt; - Long size = null; - if (i != threadCount - 1) { - size = perThreadCnt; - } - String sqlFinal; - if (size != null) { - sqlFinal = sql + " LIMIT " + offset + "," + size; - } else { - sqlFinal = sql + " LIMIT " + offset + "," + cnt; - } - Future future = executor.submit( - () -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg)); - futures.add(future); - } - - for (Future future : futures) { - future.get(); - } - - executor.shutdown(); - } else { - executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg); - } - - logger.info(hbaseMapping.getHbaseTable() + " etl completed in: " - + (System.currentTimeMillis() - start) / 1000 + "s!"); - - etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); - } - - if (errMsg.isEmpty()) { - etlResult.setSucceeded(true); - } else { - etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); - } - return etlResult; - } - - /** - * 执行导入 - * - * @param ds - * @param sql - * @param hbaseMapping - * @param hbaseTemplate - * @param successCount - * @param errMsg - * @return - */ - private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping, - HbaseTemplate hbaseTemplate, AtomicLong successCount, List errMsg) { - try { - Util.sqlRS(ds, sql, rs -> { - int i = 1; - - try { - boolean complete = false; - List rows = new ArrayList<>(); - String[] rowKeyColumns = null; - if (hbaseMapping.getRowKey() != null) { - rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); - } - while (rs.next()) { - int cc = rs.getMetaData().getColumnCount(); - int[] jdbcTypes = new int[cc]; - Class[] classes = new Class[cc]; - for (int j = 1; j <= cc; j++) { - int jdbcType = rs.getMetaData().getColumnType(j); - jdbcTypes[j - 1] = jdbcType; - classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType); - } - HRow row = new HRow(); - - if (rowKeyColumns != null) { - // 取rowKey字段拼接 - StringBuilder rowKeyVale = new StringBuilder(); - for (String rowKeyColumnName : rowKeyColumns) { - Object obj = rs.getObject(rowKeyColumnName); - if (obj != null) { - rowKeyVale.append(obj.toString()); - } - rowKeyVale.append("|"); - } - int len = rowKeyVale.length(); - if (len > 0) { - rowKeyVale.delete(len - 1, len); - } - row.setRowKey(Bytes.toBytes(rowKeyVale.toString())); - } - - for (int j = 1; j <= cc; j++) { - String columnName = rs.getMetaData().getColumnName(j); - - Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]); - if (val == null) { - continue; - } - - MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName); - // 没有配置映射 - if (columnItem == null) { - String family = hbaseMapping.getFamily(); - String qualifile = columnName; - if (hbaseMapping.isUppercaseQualifier()) { - qualifile = qualifile.toUpperCase(); - } - if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { - if (hbaseMapping.getRowKey() == null && j == 1) { - row.setRowKey(Bytes.toBytes(val.toString())); - } else { - row.addCell(family, qualifile, Bytes.toBytes(val.toString())); - } - } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { - Type type = Type.getType(classes[j - 1]); - if (hbaseMapping.getRowKey() == null && j == 1) { - row.setRowKey(TypeUtil.toBytes(val, type)); - } else { - row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); - } - } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { - PhType phType = PhType.getType(classes[j - 1]); - if (hbaseMapping.getRowKey() == null && j == 1) { - row.setRowKey(PhTypeUtil.toBytes(val, phType)); - } else { - row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); - } - } - } else { - // 如果不需要类型转换 - if (columnItem.getType() == null || "".equals(columnItem.getType())) { - if (val instanceof java.sql.Date) { - SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd"); - val = dateFmt.format((Date) val); - } else if (val instanceof Timestamp) { - SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - val = datetimeFmt.format((Date) val); - } - - byte[] valBytes = Bytes.toBytes(val.toString()); - if (columnItem.isRowKey()) { - if (columnItem.getRowKeyLen() != null) { - valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); - row.setRowKey(valBytes); - } else { - row.setRowKey(valBytes); - } - } else { - row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); - } - } else { - if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { - byte[] valBytes = Bytes.toBytes(val.toString()); - if (columnItem.isRowKey()) { - if (columnItem.getRowKeyLen() != null) { - valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); - } - row.setRowKey(valBytes); - } else { - row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); - } - } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { - Type type = Type.getType(columnItem.getType()); - if (columnItem.isRowKey()) { - if (columnItem.getRowKeyLen() != null) { - String v = limitLenNum(columnItem.getRowKeyLen(), val); - row.setRowKey(Bytes.toBytes(v)); - } else { - row.setRowKey(TypeUtil.toBytes(val, type)); - } - } else { - row.addCell(columnItem.getFamily(), - columnItem.getQualifier(), - TypeUtil.toBytes(val, type)); - } - } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { - PhType phType = PhType.getType(columnItem.getType()); - if (columnItem.isRowKey()) { - row.setRowKey(PhTypeUtil.toBytes(val, phType)); - } else { - row.addCell(columnItem.getFamily(), - columnItem.getQualifier(), - PhTypeUtil.toBytes(val, phType)); - } - } - } - } - } - - if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空"); - - rows.add(row); - complete = false; - if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); - rows.clear(); - complete = true; - } - i++; - successCount.incrementAndGet(); - if (logger.isDebugEnabled()) { - logger.debug("successful import count:" + successCount.get()); - } - } - - if (!complete && !rows.isEmpty()) { - hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); - } - - } catch (Exception e) { - logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); - errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage()); - // throw new RuntimeException(e); - } - return i; - }); - return true; - } catch (Exception e) { - logger.error(e.getMessage(), e); - return false; - } - } - - private static String limitLenNum(int len, Object val) { - if (val == null) { - return null; - } - if (val instanceof Number) { - return String.format("%0" + len + "d", (Number) ((Number) val).longValue()); - } else if (val instanceof String) { - return String.format("%0" + len + "d", Long.parseLong((String) val)); - } - return null; - } -} +package com.alibaba.otter.canal.client.adapter.hbase.service; + +import java.sql.ResultSetMetaData; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import javax.sql.DataSource; + +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.hbase.support.HRow; +import com.alibaba.otter.canal.client.adapter.hbase.support.HbaseTemplate; +import com.alibaba.otter.canal.client.adapter.hbase.support.PhType; +import com.alibaba.otter.canal.client.adapter.hbase.support.PhTypeUtil; +import com.alibaba.otter.canal.client.adapter.hbase.support.Type; +import com.alibaba.otter.canal.client.adapter.hbase.support.TypeUtil; +import com.alibaba.otter.canal.client.adapter.support.EtlResult; +import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil; +import com.alibaba.otter.canal.client.adapter.support.Util; +import com.google.common.base.Joiner; + +/** + * HBase ETL 操作业务类 + * + * @author rewerma @ 2018-10-20 + * @version 1.0.0 + */ +public class HbaseEtlService { + + private static Logger logger = LoggerFactory.getLogger(HbaseEtlService.class); + + + /** + * 建表 + * + * @param hbaseTemplate + * @param config + */ + public static void createTable(HbaseTemplate hbaseTemplate, MappingConfig config) { + try { + // 判断hbase表是否存在，不存在则建表 + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); + if (!hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { + hbaseTemplate.createTable(hbaseMapping.getHbaseTable(), hbaseMapping.getFamily()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + + /** + * 导入数据 + * + * @param ds 数据源 + * @param hbaseTemplate hbaseTemplate + * @param config 配置 + * @param params 筛选条件 + * @return 导入结果 + */ + public static EtlResult importData(DataSource ds, HbaseTemplate hbaseTemplate, MappingConfig config, + List params) { + EtlResult etlResult = new EtlResult(); + AtomicLong successCount = new AtomicLong(); + List errMsg = new ArrayList<>(); + String hbaseTable = ""; + try { + if (config == null) { + logger.error("Config is null!"); + etlResult.setSucceeded(false); + etlResult.setErrorMessage("Config is null!"); + return etlResult; + } + MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping(); + hbaseTable = hbaseMapping.getHbaseTable(); + + long start = System.currentTimeMillis(); + + if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) { + logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!"); + // 如果表存在则删除 + if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) { + hbaseTemplate.disableTable(hbaseMapping.getHbaseTable()); + hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable()); + } + params = null; + } else { + logger.info(hbaseMapping.getHbaseTable() + " etl is starting!"); + } + createTable(hbaseTemplate, config); + + // 拼接sql + String sql = "SELECT * FROM " + config.getHbaseMapping().getDatabase() + "." + hbaseMapping.getTable(); + + // 拼接条件 + if (params != null && params.size() == 1 && hbaseMapping.getEtlCondition() == null) { + AtomicBoolean stExists = new AtomicBoolean(false); + // 验证是否有SYS_TIME字段 + Util.sqlRS(ds, sql, rs -> { + try { + ResultSetMetaData rsmd = rs.getMetaData(); + int cnt = rsmd.getColumnCount(); + for (int i = 1; i <= cnt; i++) { + String columnName = rsmd.getColumnName(i); + if ("SYS_TIME".equalsIgnoreCase(columnName)) { + stExists.set(true); + break; + } + } + } catch (Exception e) { + // ignore + } + return null; + }); + if (stExists.get()) { + sql += " WHERE SYS_TIME >= '" + params.get(0) + "' "; + } + } else if (hbaseMapping.getEtlCondition() != null && params != null) { + String etlCondition = hbaseMapping.getEtlCondition(); + int size = params.size(); + for (int i = 0; i < size; i++) { + etlCondition = etlCondition.replace("{" + i + "}", params.get(i)); + } + + sql += " " + etlCondition; + } + + // 获取总数 + String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT "; + long cnt = (Long) Util.sqlRS(ds, countSql, rs -> { + Long count = null; + try { + if (rs.next()) { + count = ((Number) rs.getObject(1)).longValue(); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return count == null ? 0 : count; + }); + + // 当大于1万条记录时开启多线程 + if (cnt >= 10000) { + int threadCount = 3; + long perThreadCnt = cnt / threadCount; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + List> futures = new ArrayList<>(threadCount); + for (int i = 0; i < threadCount; i++) { + long offset = i * perThreadCnt; + Long size = null; + if (i != threadCount - 1) { + size = perThreadCnt; + } + String sqlFinal; + if (size != null) { + sqlFinal = sql + " LIMIT " + offset + "," + size; + } else { + sqlFinal = sql + " LIMIT " + offset + "," + cnt; + } + Future future = executor.submit( + () -> executeSqlImport(ds, sqlFinal, hbaseMapping, hbaseTemplate, successCount, errMsg)); + futures.add(future); + } + + for (Future future : futures) { + future.get(); + } + + executor.shutdown(); + } else { + executeSqlImport(ds, sql, hbaseMapping, hbaseTemplate, successCount, errMsg); + } + + logger.info(hbaseMapping.getHbaseTable() + " etl completed in: " + + (System.currentTimeMillis() - start) / 1000 + "s!"); + + etlResult.setResultMessage("导入HBase表 " + hbaseMapping.getHbaseTable() + " 数据：" + successCount.get() + " 条"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); + } + + if (errMsg.isEmpty()) { + etlResult.setSucceeded(true); + } else { + etlResult.setErrorMessage(Joiner.on("\n").join(errMsg)); + } + return etlResult; + } + + /** + * 执行导入 + * + * @param ds + * @param sql + * @param hbaseMapping + * @param hbaseTemplate + * @param successCount + * @param errMsg + * @return + */ + private static boolean executeSqlImport(DataSource ds, String sql, MappingConfig.HbaseMapping hbaseMapping, + HbaseTemplate hbaseTemplate, AtomicLong successCount, List errMsg) { + try { + Util.sqlRS(ds, sql, rs -> { + int i = 1; + + try { + boolean complete = false; + List rows = new ArrayList<>(); + String[] rowKeyColumns = null; + if (hbaseMapping.getRowKey() != null) { + rowKeyColumns = hbaseMapping.getRowKey().trim().split(","); + } + while (rs.next()) { + int cc = rs.getMetaData().getColumnCount(); + int[] jdbcTypes = new int[cc]; + Class[] classes = new Class[cc]; + for (int j = 1; j <= cc; j++) { + int jdbcType = rs.getMetaData().getColumnType(j); + jdbcTypes[j - 1] = jdbcType; + classes[j - 1] = JdbcTypeUtil.jdbcType2javaType(jdbcType); + } + HRow row = new HRow(); + + if (rowKeyColumns != null) { + // 取rowKey字段拼接 + StringBuilder rowKeyVale = new StringBuilder(); + for (String rowKeyColumnName : rowKeyColumns) { + Object obj = rs.getObject(rowKeyColumnName); + if (obj != null) { + rowKeyVale.append(obj.toString()); + } + rowKeyVale.append("|"); + } + int len = rowKeyVale.length(); + if (len > 0) { + rowKeyVale.delete(len - 1, len); + } + row.setRowKey(Bytes.toBytes(rowKeyVale.toString())); + } + + for (int j = 1; j <= cc; j++) { + String columnName = rs.getMetaData().getColumnName(j); + + Object val = JdbcTypeUtil.getRSData(rs, columnName, jdbcTypes[j - 1]); + if (val == null) { + continue; + } + + MappingConfig.ColumnItem columnItem = hbaseMapping.getColumnItems().get(columnName); + // 没有配置映射 + if (columnItem == null) { + String family = hbaseMapping.getFamily(); + String qualifile = columnName; + if (hbaseMapping.isUppercaseQualifier()) { + qualifile = qualifile.toUpperCase(); + } + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { + if (hbaseMapping.getRowKey() == null && j == 1) { + row.setRowKey(Bytes.toBytes(val.toString())); + } else { + row.addCell(family, qualifile, Bytes.toBytes(val.toString())); + } + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { + Type type = Type.getType(classes[j - 1]); + if (hbaseMapping.getRowKey() == null && j == 1) { + row.setRowKey(TypeUtil.toBytes(val, type)); + } else { + row.addCell(family, qualifile, TypeUtil.toBytes(val, type)); + } + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { + PhType phType = PhType.getType(classes[j - 1]); + if (hbaseMapping.getRowKey() == null && j == 1) { + row.setRowKey(PhTypeUtil.toBytes(val, phType)); + } else { + row.addCell(family, qualifile, PhTypeUtil.toBytes(val, phType)); + } + } + } else { + // 如果不需要类型转换 + if (columnItem.getType() == null || "".equals(columnItem.getType())) { + if (val instanceof java.sql.Date) { + SimpleDateFormat dateFmt = new SimpleDateFormat("yyyy-MM-dd"); + val = dateFmt.format((Date) val); + } else if (val instanceof Timestamp) { + SimpleDateFormat datetimeFmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + val = datetimeFmt.format((Date) val); + } + + byte[] valBytes = Bytes.toBytes(val.toString()); + if (columnItem.isRowKey()) { + if (columnItem.getRowKeyLen() != null) { + valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); + row.setRowKey(valBytes); + } else { + row.setRowKey(valBytes); + } + } else { + row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); + } + } else { + if (MappingConfig.Mode.STRING == hbaseMapping.getMode()) { + byte[] valBytes = Bytes.toBytes(val.toString()); + if (columnItem.isRowKey()) { + if (columnItem.getRowKeyLen() != null) { + valBytes = Bytes.toBytes(limitLenNum(columnItem.getRowKeyLen(), val)); + } + row.setRowKey(valBytes); + } else { + row.addCell(columnItem.getFamily(), columnItem.getQualifier(), valBytes); + } + } else if (MappingConfig.Mode.NATIVE == hbaseMapping.getMode()) { + Type type = Type.getType(columnItem.getType()); + if (columnItem.isRowKey()) { + if (columnItem.getRowKeyLen() != null) { + String v = limitLenNum(columnItem.getRowKeyLen(), val); + row.setRowKey(Bytes.toBytes(v)); + } else { + row.setRowKey(TypeUtil.toBytes(val, type)); + } + } else { + row.addCell(columnItem.getFamily(), + columnItem.getQualifier(), + TypeUtil.toBytes(val, type)); + } + } else if (MappingConfig.Mode.PHOENIX == hbaseMapping.getMode()) { + PhType phType = PhType.getType(columnItem.getType()); + if (columnItem.isRowKey()) { + row.setRowKey(PhTypeUtil.toBytes(val, phType)); + } else { + row.addCell(columnItem.getFamily(), + columnItem.getQualifier(), + PhTypeUtil.toBytes(val, phType)); + } + } + } + } + } + + if (row.getRowKey() == null) throw new RuntimeException("RowKey 值为空"); + + rows.add(row); + complete = false; + if (i % hbaseMapping.getCommitBatch() == 0 && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); + rows.clear(); + complete = true; + } + i++; + successCount.incrementAndGet(); + if (logger.isDebugEnabled()) { + logger.debug("successful import count:" + successCount.get()); + } + } + + if (!complete && !rows.isEmpty()) { + hbaseTemplate.puts(hbaseMapping.getHbaseTable(), rows); + } + + } catch (Exception e) { + logger.error(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage(), e); + errMsg.add(hbaseMapping.getHbaseTable() + " etl failed! ==>" + e.getMessage()); + // throw new RuntimeException(e); + } + return i; + }); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + } + + private static String limitLenNum(int len, Object val) { + if (val == null) { + return null; + } + if (val instanceof Number) { + return String.format("%0" + len + "d", (Number) ((Number) val).longValue()); + } else if (val instanceof String) { + return String.format("%0" + len + "d", Long.parseLong((String) val)); + } + return null; + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 7d082bc6..12d5a040 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -1,216 +1,216 @@ -package com.alibaba.otter.canal.adapter.launcher.loader; - -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.env.EnumerablePropertySource; -import org.springframework.core.env.Environment; -import org.springframework.core.env.PropertySource; -import org.springframework.core.env.StandardEnvironment; - -import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; -import com.alibaba.otter.canal.client.adapter.OuterAdapter; -import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; -import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; - -/** - * 外部适配器的加载器 - * - * @version 1.0.0 - */ -public class CanalAdapterLoader { - - private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); - - private CanalClientConfig canalClientConfig; - - private Map canalWorkers = new HashMap<>(); - - private Map canalMQWorker = new HashMap<>(); - - private ExtensionLoader loader; - - public CanalAdapterLoader(CanalClientConfig canalClientConfig){ - this.canalClientConfig = canalClientConfig; - } - - /** - * 初始化canal-client - */ - public void init() { - loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); - - String canalServerHost = this.canalClientConfig.getCanalServerHost(); - SocketAddress sa = null; - if (canalServerHost != null) { - String[] ipPort = canalServerHost.split(":"); - sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); - } - String zkHosts = this.canalClientConfig.getZookeeperHosts(); - - if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - - for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { - List canalOutConnectors = new ArrayList<>(); - for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { - loadAdapter(c, canalOutConnectors); - } - canalOuterAdapterGroups.add(canalOutConnectors); - } - CanalAdapterWorker worker; - if (sa != null) { - worker = new CanalAdapterWorker(canalClientConfig, - canalAdapter.getInstance(), - sa, - canalOuterAdapterGroups); - } else if (zkHosts != null) { - worker = new CanalAdapterWorker(canalClientConfig, - canalAdapter.getInstance(), - zkHosts, - canalOuterAdapterGroups); - } else { - throw new RuntimeException("No canal server connector found"); - } - canalWorkers.put(canalAdapter.getInstance(), worker); - worker.start(); - logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); - } - } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client-kafka的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { - for (CanalClientConfig.Group group : canalAdapter.getGroups()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - List canalOuterAdapters = new ArrayList<>(); - for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadAdapter(config, canalOuterAdapters); - } - canalOuterAdapterGroups.add(canalOuterAdapters); - - CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, - canalClientConfig.getMqServers(), - canalAdapter.getInstance(), - group.getGroupId(), - canalOuterAdapterGroups, - canalClientConfig.getFlatMessage()); - canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); - canalKafkaWorker.start(); - logger.info("Start adapter for canal-client mq topic: {} succeed", - canalAdapter.getInstance() + "-" + group.getGroupId()); - } - } - } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { - // 初始化canal-client-rocketMQ的适配器 - for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { - for (CanalClientConfig.Group group : canalAdapter.getGroups()) { - List> canalOuterAdapterGroups = new ArrayList<>(); - List canalOuterAdapters = new ArrayList<>(); - for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadAdapter(config, canalOuterAdapters); - } - canalOuterAdapterGroups.add(canalOuterAdapters); - CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, - canalClientConfig.getMqServers(), - canalAdapter.getInstance(), - group.getGroupId(), - canalOuterAdapterGroups, - canalClientConfig.getAccessKey(), - canalClientConfig.getSecretKey(), - canalClientConfig.getFlatMessage()); - canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); - rocketMQWorker.start(); - - logger.info("Start adapter for canal-client mq topic: {} succeed", - canalAdapter.getInstance() + "-" + group.getGroupId()); - } - } - } - } - - private void loadAdapter(OuterAdapterConfig config, List canalOutConnectors) { - try { - OuterAdapter adapter; - adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey())); - - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - // 替换ClassLoader - Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); - Environment env = (Environment) SpringContext.getBean(Environment.class); - Properties evnProperties = null; - if (env instanceof StandardEnvironment) { - evnProperties = new Properties(); - for (PropertySource propertySource : ((StandardEnvironment) env).getPropertySources()) { - if (propertySource instanceof EnumerablePropertySource) { - String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames(); - for (String name : names) { - Object val = propertySource.getProperty(name); - if (val != null) { - evnProperties.put(name, val); - } - } - } - } - } - adapter.init(config, evnProperties); - Thread.currentThread().setContextClassLoader(cl); - canalOutConnectors.add(adapter); - logger.info("Load canal adapter: {} succeed", config.getName()); - } catch (Exception e) { - logger.error("Load canal adapter: {} failed", config.getName(), e); - } - } - - /** - * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 - */ - public void destroy() { - if (!canalWorkers.isEmpty()) { - ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); - List> futures = new ArrayList<>(); - for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { - futures.add(stopExecutorService.submit(() -> { - canalAdapterWorker.stop(); - return true; - })); - } - futures.forEach(future -> { - try { - future.get(); - } catch (Exception e) { - // ignore - } - }); - stopExecutorService.shutdown(); - } - - if (!canalMQWorker.isEmpty()) { - ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size()); - List> futures = new ArrayList<>(); - for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) { - futures.add(stopMQWorkerService.submit(() -> { - canalAdapterMQWorker.stop(); - return true; - })); - } - futures.forEach(future -> { - try { - future.get(); - } catch (Exception e) { - // ignore - } - }); - stopMQWorkerService.shutdown(); - } - logger.info("All canal adapters destroyed"); - } -} +package com.alibaba.otter.canal.adapter.launcher.loader; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; + +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; +import com.alibaba.otter.canal.client.adapter.OuterAdapter; +import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; +import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; +import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; + +/** + * 外部适配器的加载器 + * + * @version 1.0.0 + */ +public class CanalAdapterLoader { + + private static final Logger logger = LoggerFactory.getLogger(CanalAdapterLoader.class); + + private CanalClientConfig canalClientConfig; + + private Map canalWorkers = new HashMap<>(); + + private Map canalMQWorker = new HashMap<>(); + + private ExtensionLoader loader; + + public CanalAdapterLoader(CanalClientConfig canalClientConfig){ + this.canalClientConfig = canalClientConfig; + } + + /** + * 初始化canal-client + */ + public void init() { + loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class); + + String canalServerHost = this.canalClientConfig.getCanalServerHost(); + SocketAddress sa = null; + if (canalServerHost != null) { + String[] ipPort = canalServerHost.split(":"); + sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1])); + } + String zkHosts = this.canalClientConfig.getZookeeperHosts(); + + if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) { + // 初始化canal-client的适配器 + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + + for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { + List canalOutConnectors = new ArrayList<>(); + for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { + loadAdapter(c, canalOutConnectors); + } + canalOuterAdapterGroups.add(canalOutConnectors); + } + CanalAdapterWorker worker; + if (sa != null) { + worker = new CanalAdapterWorker(canalClientConfig, + canalAdapter.getInstance(), + sa, + canalOuterAdapterGroups); + } else if (zkHosts != null) { + worker = new CanalAdapterWorker(canalClientConfig, + canalAdapter.getInstance(), + zkHosts, + canalOuterAdapterGroups); + } else { + throw new RuntimeException("No canal server connector found"); + } + canalWorkers.put(canalAdapter.getInstance(), worker); + worker.start(); + logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance()); + } + } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) { + // 初始化canal-client-kafka的适配器 + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + for (CanalClientConfig.Group group : canalAdapter.getGroups()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + List canalOuterAdapters = new ArrayList<>(); + for (OuterAdapterConfig config : group.getOuterAdapters()) { + loadAdapter(config, canalOuterAdapters); + } + canalOuterAdapterGroups.add(canalOuterAdapters); + + CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig, + canalClientConfig.getMqServers(), + canalAdapter.getInstance(), + group.getGroupId(), + canalOuterAdapterGroups, + canalClientConfig.getFlatMessage()); + canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); + canalKafkaWorker.start(); + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); + } + } + } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { + // 初始化canal-client-rocketMQ的适配器 + for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) { + for (CanalClientConfig.Group group : canalAdapter.getGroups()) { + List> canalOuterAdapterGroups = new ArrayList<>(); + List canalOuterAdapters = new ArrayList<>(); + for (OuterAdapterConfig config : group.getOuterAdapters()) { + loadAdapter(config, canalOuterAdapters); + } + canalOuterAdapterGroups.add(canalOuterAdapters); + CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, + canalClientConfig.getMqServers(), + canalAdapter.getInstance(), + group.getGroupId(), + canalOuterAdapterGroups, + canalClientConfig.getAccessKey(), + canalClientConfig.getSecretKey(), + canalClientConfig.getFlatMessage()); + canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); + rocketMQWorker.start(); + + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); + } + } + } + } + + private void loadAdapter(OuterAdapterConfig config, List canalOutConnectors) { + try { + OuterAdapter adapter; + adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey())); + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + // 替换ClassLoader + Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); + Environment env = (Environment) SpringContext.getBean(Environment.class); + Properties evnProperties = null; + if (env instanceof StandardEnvironment) { + evnProperties = new Properties(); + for (PropertySource propertySource : ((StandardEnvironment) env).getPropertySources()) { + if (propertySource instanceof EnumerablePropertySource) { + String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames(); + for (String name : names) { + Object val = propertySource.getProperty(name); + if (val != null) { + evnProperties.put(name, val); + } + } + } + } + } + adapter.init(config, evnProperties); + Thread.currentThread().setContextClassLoader(cl); + canalOutConnectors.add(adapter); + logger.info("Load canal adapter: {} succeed", config.getName()); + } catch (Exception e) { + logger.error("Load canal adapter: {} failed", config.getName(), e); + } + } + + /** + * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁 + */ + public void destroy() { + if (!canalWorkers.isEmpty()) { + ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size()); + List> futures = new ArrayList<>(); + for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) { + futures.add(stopExecutorService.submit(() -> { + canalAdapterWorker.stop(); + return true; + })); + } + futures.forEach(future -> { + try { + future.get(); + } catch (Exception e) { + // ignore + } + }); + stopExecutorService.shutdown(); + } + + if (!canalMQWorker.isEmpty()) { + ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size()); + List> futures = new ArrayList<>(); + for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) { + futures.add(stopMQWorkerService.submit(() -> { + canalAdapterMQWorker.stop(); + return true; + })); + } + futures.forEach(future -> { + try { + future.get(); + } catch (Exception e) { + // ignore + } + }); + stopMQWorkerService.shutdown(); + } + logger.info("All canal adapters destroyed"); + } +} diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java index a9a3bb6e..400e0463 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java @@ -1,248 +1,246 @@ -package com.alibaba.otter.canal.adapter.launcher.monitor.remote; - -import java.io.FileWriter; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.otter.canal.common.utils.CommonUtils; -import com.alibaba.otter.canal.common.utils.NamedThreadFactory; -import com.google.common.base.Joiner; -import com.google.common.collect.MapMaker; - -/** - * 基于数据库的远程配置装载器 - * - * @author rewerma 2019-01-25 下午05:20:16 - * @version 1.0.0 - */ -public class DbRemoteConfigLoader implements RemoteConfigLoader { - - private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class); - - private DruidDataSource dataSource; - - private static volatile long currentConfigTimestamp = 0; - private Map remoteAdapterConfigs = new MapMaker().makeMap(); - - private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2, - new NamedThreadFactory("remote-adapter-config-scan")); - - private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl(); - - public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){ - dataSource = new DruidDataSource(); - if (StringUtils.isEmpty(driverName)) { - driverName = "com.mysql.jdbc.Driver"; - } - dataSource.setDriverClassName(driverName); - dataSource.setUrl(jdbcUrl); - dataSource.setUsername(jdbcUsername); - dataSource.setPassword(jdbcPassword); - dataSource.setInitialSize(1); - dataSource.setMinIdle(1); - dataSource.setMaxActive(1); - dataSource.setMaxWait(60000); - dataSource.setTimeBetweenEvictionRunsMillis(60000); - dataSource.setMinEvictableIdleTimeMillis(300000); - try { - dataSource.init(); - } catch (SQLException e) { - throw new RuntimeException(e.getMessage(), e); - } - } - - /** - * 加载远程application.yml配置 - */ - @Override - public void loadRemoteConfig() { - try { - // 加载远程adapter配置 - ConfigItem configItem = getRemoteAdapterConfig(); - if (configItem != null) { - if (configItem.getModifiedTime() != currentConfigTimestamp) { - currentConfigTimestamp = configItem.getModifiedTime(); - overrideLocalCanalConfig(configItem.getContent()); - logger.info("## Loaded remote adapter config: application.yml"); - } - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - /** - * 获取远程application.yml配置 - * - * @return 配置对象 - */ - private ConfigItem getRemoteAdapterConfig() { - String sql = "select name, content, modified_time from canal_config where id=2"; - try (Connection conn = dataSource.getConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - if (rs.next()) { - ConfigItem configItem = new ConfigItem(); - configItem.setId(2L); - configItem.setName(rs.getString("name")); - configItem.setContent(rs.getString("content")); - configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); - return configItem; - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - return null; - } - - /** - * 覆盖本地application.yml文件 - * - * @param content 文件内容 - */ - private void overrideLocalCanalConfig(String content) { - try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) { - writer.write(content); - writer.flush(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - /** - * 加载adapter配置 - */ - @Override - public void loadRemoteAdapterConfigs() { - try { - // 加载远程adapter配置 - loadModifiedAdapterConfigs(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - /** - * 加载有变动的adapter配置 - */ - @SuppressWarnings("unchecked") - private void loadModifiedAdapterConfigs() { - Map[] res = new Map[2]; - Map remoteConfigStatus = new HashMap<>(); - String sql = "select id, category, name, modified_time from canal_adapter_config"; - try (Connection conn = dataSource.getConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery(sql)) { - while (rs.next()) { - ConfigItem configItem = new ConfigItem(); - configItem.setId(rs.getLong("id")); - configItem.setCategory(rs.getString("category")); - configItem.setName(rs.getString("name")); - configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); - remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - - if (!remoteConfigStatus.isEmpty()) { - List changedIds = new ArrayList<>(); - - for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) { - ConfigItem currentConfig = remoteAdapterConfigs - .get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName()); - if (currentConfig == null) { - // 新增 - changedIds.add(remoteConfigStat.getId()); - } else { - // 修改 - if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) { - changedIds.add(remoteConfigStat.getId()); - } - } - } - if (!changedIds.isEmpty()) { - String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in (" - + Joiner.on(",").join(changedIds) + ")"; - try (Connection conn = dataSource.getConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery(contentsSql)) { - while (rs.next()) { - ConfigItem configItemNew = new ConfigItem(); - configItemNew.setId(rs.getLong("id")); - configItemNew.setCategory(rs.getString("category")); - configItemNew.setName(rs.getString("name")); - configItemNew.setContent(rs.getString("content")); - configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime()); - - remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(), - configItemNew); - remoteAdapterMonitor.onModify(configItemNew); - } - - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - } - - for (ConfigItem configItem : remoteAdapterConfigs.values()) { - if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) { - // 删除 - remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName()); - remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName()); - } - } - } - - /** - * 启动监听数据库变化 - */ - @Override - public void startMonitor() { - // 监听application.yml变化 - executor.scheduleWithFixedDelay(() -> { - try { - loadRemoteConfig(); - } catch (Throwable e) { - logger.error("scan remote application.yml failed", e); - } - }, 10, 3, TimeUnit.SECONDS); - - // 监听adapter变化 - executor.scheduleWithFixedDelay(() -> { - try { - loadRemoteAdapterConfigs(); - } catch (Throwable e) { - logger.error("scan remote adapter configs failed", e); - } - }, 10, 3, TimeUnit.SECONDS); - } - - /** - * 销毁 - */ - @Override - public void destroy() { - executor.shutdownNow(); - try { - dataSource.close(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -} +package com.alibaba.otter.canal.adapter.launcher.monitor.remote; + +import java.io.FileWriter; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.common.utils.CommonUtils; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.google.common.base.Joiner; +import com.google.common.collect.MapMaker; + +/** + * 基于数据库的远程配置装载器 + * + * @author rewerma 2019-01-25 下午05:20:16 + * @version 1.0.0 + */ +public class DbRemoteConfigLoader implements RemoteConfigLoader { + + private static final Logger logger = LoggerFactory.getLogger(DbRemoteConfigLoader.class); + + private DruidDataSource dataSource; + + private static volatile long currentConfigTimestamp = 0; + private Map remoteAdapterConfigs = new MapMaker().makeMap(); + + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(2, + new NamedThreadFactory("remote-adapter-config-scan")); + + private RemoteAdapterMonitor remoteAdapterMonitor = new RemoteAdapterMonitorImpl(); + + public DbRemoteConfigLoader(String driverName, String jdbcUrl, String jdbcUsername, String jdbcPassword){ + dataSource = new DruidDataSource(); + if (StringUtils.isEmpty(driverName)) { + driverName = "com.mysql.jdbc.Driver"; + } + dataSource.setDriverClassName(driverName); + dataSource.setUrl(jdbcUrl); + dataSource.setUsername(jdbcUsername); + dataSource.setPassword(jdbcPassword); + dataSource.setInitialSize(1); + dataSource.setMinIdle(1); + dataSource.setMaxActive(1); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + try { + dataSource.init(); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + /** + * 加载远程application.yml配置 + */ + @Override + public void loadRemoteConfig() { + try { + // 加载远程adapter配置 + ConfigItem configItem = getRemoteAdapterConfig(); + if (configItem != null) { + if (configItem.getModifiedTime() != currentConfigTimestamp) { + currentConfigTimestamp = configItem.getModifiedTime(); + overrideLocalCanalConfig(configItem.getContent()); + logger.info("## Loaded remote adapter config: application.yml"); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + /** + * 获取远程application.yml配置 + * + * @return 配置对象 + */ + private ConfigItem getRemoteAdapterConfig() { + String sql = "select name, content, modified_time from canal_config where id=2"; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + if (rs.next()) { + ConfigItem configItem = new ConfigItem(); + configItem.setId(2L); + configItem.setName(rs.getString("name")); + configItem.setContent(rs.getString("content")); + configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); + return configItem; + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + return null; + } + + /** + * 覆盖本地application.yml文件 + * + * @param content 文件内容 + */ + private void overrideLocalCanalConfig(String content) { + try (FileWriter writer = new FileWriter(CommonUtils.getConfPath() + "application.yml")) { + writer.write(content); + writer.flush(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + /** + * 加载adapter配置 + */ + @Override + public void loadRemoteAdapterConfigs() { + try { + // 加载远程adapter配置 + loadModifiedAdapterConfigs(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + /** + * 加载有变动的adapter配置 + */ + private void loadModifiedAdapterConfigs() { + Map remoteConfigStatus = new HashMap<>(); + String sql = "select id, category, name, modified_time from canal_adapter_config"; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) { + ConfigItem configItem = new ConfigItem(); + configItem.setId(rs.getLong("id")); + configItem.setCategory(rs.getString("category")); + configItem.setName(rs.getString("name")); + configItem.setModifiedTime(rs.getTimestamp("modified_time").getTime()); + remoteConfigStatus.put(configItem.getCategory() + "/" + configItem.getName(), configItem); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + if (!remoteConfigStatus.isEmpty()) { + List changedIds = new ArrayList<>(); + + for (ConfigItem remoteConfigStat : remoteConfigStatus.values()) { + ConfigItem currentConfig = remoteAdapterConfigs + .get(remoteConfigStat.getCategory() + "/" + remoteConfigStat.getName()); + if (currentConfig == null) { + // 新增 + changedIds.add(remoteConfigStat.getId()); + } else { + // 修改 + if (currentConfig.getModifiedTime() != remoteConfigStat.getModifiedTime()) { + changedIds.add(remoteConfigStat.getId()); + } + } + } + if (!changedIds.isEmpty()) { + String contentsSql = "select id, category, name, content, modified_time from canal_adapter_config where id in (" + + Joiner.on(",").join(changedIds) + ")"; + try (Connection conn = dataSource.getConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(contentsSql)) { + while (rs.next()) { + ConfigItem configItemNew = new ConfigItem(); + configItemNew.setId(rs.getLong("id")); + configItemNew.setCategory(rs.getString("category")); + configItemNew.setName(rs.getString("name")); + configItemNew.setContent(rs.getString("content")); + configItemNew.setModifiedTime(rs.getTimestamp("modified_time").getTime()); + + remoteAdapterConfigs.put(configItemNew.getCategory() + "/" + configItemNew.getName(), + configItemNew); + remoteAdapterMonitor.onModify(configItemNew); + } + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + + for (ConfigItem configItem : remoteAdapterConfigs.values()) { + if (!remoteConfigStatus.containsKey(configItem.getCategory() + "/" + configItem.getName())) { + // 删除 + remoteAdapterConfigs.remove(configItem.getCategory() + "/" + configItem.getName()); + remoteAdapterMonitor.onDelete(configItem.getCategory() + "/" + configItem.getName()); + } + } + } + + /** + * 启动监听数据库变化 + */ + @Override + public void startMonitor() { + // 监听application.yml变化 + executor.scheduleWithFixedDelay(() -> { + try { + loadRemoteConfig(); + } catch (Throwable e) { + logger.error("scan remote application.yml failed", e); + } + }, 10, 3, TimeUnit.SECONDS); + + // 监听adapter变化 + executor.scheduleWithFixedDelay(() -> { + try { + loadRemoteAdapterConfigs(); + } catch (Throwable e) { + logger.error("scan remote adapter configs failed", e); + } + }, 10, 3, TimeUnit.SECONDS); + } + + /** + * 销毁 + */ + @Override + public void destroy() { + executor.shutdownNow(); + try { + dataSource.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java index 514d9a79..d65e0ec5 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java @@ -1,53 +1,51 @@ -package com.alibaba.otter.canal.client.adapter.rdb.config; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; - -import com.alibaba.fastjson.JSONObject; -import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - -import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; - -/** - * RDB表映射配置加载器 - * - * @author rewerma 2018-11-07 下午02:41:34 - * @version 1.0.0 - */ -public class ConfigLoader { - - private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); - - /** - * 加载RDB表映射配置 - * - * @return 配置名/配置文件名--对象 - */ - public static Map load(Properties envProperties) { - logger.info("## Start loading rdb mapping config ... "); - - Map result = new LinkedHashMap<>(); - - Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); - configContentMap.forEach((fileName, content) -> { - MappingConfig config = YmlConfigBinder - .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); - if (config == null) { - return; - } - try { - config.validate(); - } catch (Exception e) { - throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); - } - result.put(fileName, config); - }); - - logger.info("## Rdb mapping config loaded"); - return result; - } -} +package com.alibaba.otter.canal.client.adapter.rdb.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; + +/** + * RDB表映射配置加载器 + * + * @author rewerma 2018-11-07 下午02:41:34 + * @version 1.0.0 + */ +public class ConfigLoader { + + private static Logger logger = LoggerFactory.getLogger(ConfigLoader.class); + + /** + * 加载RDB表映射配置 + * + * @return 配置名/配置文件名--对象 + */ + public static Map load(Properties envProperties) { + logger.info("## Start loading rdb mapping config ... "); + + Map result = new LinkedHashMap<>(); + + Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); + configContentMap.forEach((fileName, content) -> { + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); + if (config == null) { + return; + } + try { + config.validate(); + } catch (Exception e) { + throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e); + } + result.put(fileName, config); + }); + + logger.info("## Rdb mapping config loaded"); + return result; + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java index ddf47138..72529a37 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/monitor/RdbConfigMonitor.java @@ -1,170 +1,169 @@ -package com.alibaba.otter.canal.client.adapter.rdb.monitor; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; -import org.apache.commons.io.monitor.FileAlterationMonitor; -import org.apache.commons.io.monitor.FileAlterationObserver; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; - -import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; -import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; -import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; -import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; -import com.alibaba.otter.canal.client.adapter.support.Util; - -public class RdbConfigMonitor { - - private static final Logger logger = LoggerFactory.getLogger(RdbConfigMonitor.class); - - private static final String adapterName = "rdb"; - - private String key; - - private RdbAdapter rdbAdapter; - - private FileAlterationMonitor fileMonitor; - - public void init(String key, RdbAdapter rdbAdapter) { - this.key = key; - this.rdbAdapter = rdbAdapter; - File confDir = Util.getConfDirPath(adapterName); - try { - FileAlterationObserver observer = new FileAlterationObserver(confDir, - FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); - FileListener listener = new FileListener(); - observer.addListener(listener); - fileMonitor = new FileAlterationMonitor(3000, observer); - fileMonitor.start(); - - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - public void destroy() { - try { - fileMonitor.stop(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private class FileListener extends FileAlterationListenerAdaptor { - - @Override - public void onFileCreate(File file) { - super.onFileCreate(file); - try { - // 加载新增的配置文件 - String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); - MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); - config.validate(); - if ((key == null && config.getOuterAdapterKey() == null) - || (key != null && key.equals(config.getOuterAdapterKey()))) { - addConfigToCache(file, config); - - logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName()); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public void onFileChange(File file) { - super.onFileChange(file); - - try { - if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { - // 加载配置文件 - String configContent = MappingConfigsLoader - .loadConfig(adapterName + File.separator + file.getName()); - if (configContent == null) { - onFileDelete(file); - return; - } - MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); - config.validate(); - if ((key == null && config.getOuterAdapterKey() == null) - || (key != null && key.equals(config.getOuterAdapterKey()))) { - if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { - deleteConfigFromCache(file); - } - addConfigToCache(file, config); - } else { - // 不能修改outerAdapterKey - throw new RuntimeException("Outer adapter key not allowed modify"); - } - logger.info("Change a rdb mapping config: {} of canal adapter", file.getName()); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - @Override - public void onFileDelete(File file) { - super.onFileDelete(file); - - try { - if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { - deleteConfigFromCache(file); - - logger.info("Delete a rdb mapping config: {} of canal adapter", file.getName()); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } - - private void addConfigToCache(File file, MappingConfig mappingConfig) { - if (mappingConfig == null || mappingConfig.getDbMapping() == null) { - return; - } - rdbAdapter.getRdbMapping().put(file.getName(), mappingConfig); - if (!mappingConfig.getDbMapping().getMirrorDb()) { - Map configMap = rdbAdapter.getMappingConfigCache() - .computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." - + mappingConfig.getDbMapping().getDatabase() + "." - + mappingConfig.getDbMapping().getTable(), - k1 -> new HashMap<>()); - configMap.put(file.getName(), mappingConfig); - } else { - Map mirrorDbConfigCache = rdbAdapter.getMirrorDbConfigCache(); - mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." - + mappingConfig.getDbMapping().getDatabase(), - MirrorDbConfig.create(file.getName(), mappingConfig)); - } - } - - private void deleteConfigFromCache(File file) { - MappingConfig mappingConfig = rdbAdapter.getRdbMapping().remove(file.getName()); - - if (mappingConfig == null || mappingConfig.getDbMapping() == null) { - return; - } - if (!mappingConfig.getDbMapping().getMirrorDb()) { - for (Map configMap : rdbAdapter.getMappingConfigCache().values()) { - if (configMap != null) { - configMap.remove(file.getName()); - } - } - } else { - rdbAdapter.getMirrorDbConfigCache().forEach((key, mirrorDbConfig) -> { - if (mirrorDbConfig.getFileName().equals(file.getName())) { - rdbAdapter.getMirrorDbConfigCache().remove(key); - } - }); - } - - } - } -} +package com.alibaba.otter.canal.client.adapter.rdb.monitor; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.monitor.FileAlterationListenerAdaptor; +import org.apache.commons.io.monitor.FileAlterationMonitor; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.yaml.snakeyaml.Yaml; + +import com.alibaba.otter.canal.client.adapter.rdb.RdbAdapter; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MirrorDbConfig; +import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; +import com.alibaba.otter.canal.client.adapter.support.Util; + +public class RdbConfigMonitor { + + private static final Logger logger = LoggerFactory.getLogger(RdbConfigMonitor.class); + + private static final String adapterName = "rdb"; + + private String key; + + private RdbAdapter rdbAdapter; + + private FileAlterationMonitor fileMonitor; + + public void init(String key, RdbAdapter rdbAdapter) { + this.key = key; + this.rdbAdapter = rdbAdapter; + File confDir = Util.getConfDirPath(adapterName); + try { + FileAlterationObserver observer = new FileAlterationObserver(confDir, + FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml"))); + FileListener listener = new FileListener(); + observer.addListener(listener); + fileMonitor = new FileAlterationMonitor(3000, observer); + fileMonitor.start(); + + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + public void destroy() { + try { + fileMonitor.stop(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private class FileListener extends FileAlterationListenerAdaptor { + + @Override + public void onFileCreate(File file) { + super.onFileCreate(file); + try { + // 加载新增的配置文件 + String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName()); + MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); + config.validate(); + if ((key == null && config.getOuterAdapterKey() == null) + || (key != null && key.equals(config.getOuterAdapterKey()))) { + addConfigToCache(file, config); + + logger.info("Add a new rdb mapping config: {} to canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileChange(File file) { + super.onFileChange(file); + + try { + if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { + // 加载配置文件 + String configContent = MappingConfigsLoader + .loadConfig(adapterName + File.separator + file.getName()); + if (configContent == null) { + onFileDelete(file); + return; + } + MappingConfig config = new Yaml().loadAs(configContent, MappingConfig.class); + config.validate(); + if ((key == null && config.getOuterAdapterKey() == null) + || (key != null && key.equals(config.getOuterAdapterKey()))) { + if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { + deleteConfigFromCache(file); + } + addConfigToCache(file, config); + } else { + // 不能修改outerAdapterKey + throw new RuntimeException("Outer adapter key not allowed modify"); + } + logger.info("Change a rdb mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + @Override + public void onFileDelete(File file) { + super.onFileDelete(file); + + try { + if (rdbAdapter.getRdbMapping().containsKey(file.getName())) { + deleteConfigFromCache(file); + + logger.info("Delete a rdb mapping config: {} of canal adapter", file.getName()); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + + private void addConfigToCache(File file, MappingConfig mappingConfig) { + if (mappingConfig == null || mappingConfig.getDbMapping() == null) { + return; + } + rdbAdapter.getRdbMapping().put(file.getName(), mappingConfig); + if (!mappingConfig.getDbMapping().getMirrorDb()) { + Map configMap = rdbAdapter.getMappingConfigCache() + .computeIfAbsent(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + + mappingConfig.getDbMapping().getDatabase() + "." + + mappingConfig.getDbMapping().getTable(), + k1 -> new HashMap<>()); + configMap.put(file.getName(), mappingConfig); + } else { + Map mirrorDbConfigCache = rdbAdapter.getMirrorDbConfigCache(); + mirrorDbConfigCache.put(StringUtils.trimToEmpty(mappingConfig.getDestination()) + "." + + mappingConfig.getDbMapping().getDatabase(), + MirrorDbConfig.create(file.getName(), mappingConfig)); + } + } + + private void deleteConfigFromCache(File file) { + MappingConfig mappingConfig = rdbAdapter.getRdbMapping().remove(file.getName()); + + if (mappingConfig == null || mappingConfig.getDbMapping() == null) { + return; + } + if (!mappingConfig.getDbMapping().getMirrorDb()) { + for (Map configMap : rdbAdapter.getMappingConfigCache().values()) { + if (configMap != null) { + configMap.remove(file.getName()); + } + } + } else { + rdbAdapter.getMirrorDbConfigCache().forEach((key, mirrorDbConfig) -> { + if (mirrorDbConfig.getFileName().equals(file.getName())) { + rdbAdapter.getMirrorDbConfigCache().remove(key); + } + }); + } + + } + } +} diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java index cc85b6b1..2c3ecfa9 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbSyncService.java @@ -1,499 +1,506 @@ -package com.alibaba.otter.canal.client.adapter.rdb.service; - -import java.sql.Connection; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.*; -import java.util.function.Function; - -import javax.sql.DataSource; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.serializer.SerializerFeature; -import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; -import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping; -import com.alibaba.otter.canal.client.adapter.rdb.support.BatchExecutor; -import com.alibaba.otter.canal.client.adapter.rdb.support.SingleDml; -import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.Util; - -/** - * RDB同步操作业务 - * - * @author rewerma 2018-11-7 下午06:45:49 - * @version 1.0.0 - */ -public class RdbSyncService { - - private static final Logger logger = LoggerFactory.getLogger(RdbSyncService.class); - - // 源库表字段类型缓存: instance.schema.table -> - private Map> columnsTypeCache; - - private int threads = 3; - private boolean skipDupException; - - private List[] dmlsPartition; - private BatchExecutor[] batchExecutors; - private ExecutorService[] executorThreads; - - public List[] getDmlsPartition() { - return dmlsPartition; - } - - public Map> getColumnsTypeCache() { - return columnsTypeCache; - } - - @SuppressWarnings("unchecked") - public RdbSyncService(DataSource dataSource, Integer threads, boolean skipDupException){ - this(dataSource, threads, new ConcurrentHashMap<>(), skipDupException); - } - - @SuppressWarnings("unchecked") - public RdbSyncService(DataSource dataSource, Integer threads, Map> columnsTypeCache, - boolean skipDupException){ - this.columnsTypeCache = columnsTypeCache; - this.skipDupException = skipDupException; - try { - if (threads != null) { - this.threads = threads; - } - this.dmlsPartition = new List[this.threads]; - this.batchExecutors = new BatchExecutor[this.threads]; - this.executorThreads = new ExecutorService[this.threads]; - for (int i = 0; i < this.threads; i++) { - dmlsPartition[i] = new ArrayList<>(); - batchExecutors[i] = new BatchExecutor(dataSource); - executorThreads[i] = Executors.newSingleThreadExecutor(); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * 批量同步回调 - * - * @param dmls 批量 DML - * @param function 回调方法 - */ - public void sync(List dmls, Function function) { - try { - boolean toExecute = false; - for (Dml dml : dmls) { - if (!toExecute) { - toExecute = function.apply(dml); - } else { - function.apply(dml); - } - } - if (toExecute) { - List futures = new ArrayList<>(); - for (int i = 0; i < threads; i++) { - int j = i; - futures.add(executorThreads[i].submit(() -> { - try { - dmlsPartition[j] - .forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); - dmlsPartition[j].clear(); - batchExecutors[j].commit(); - return true; - } catch (Throwable e) { - batchExecutors[j].rollback(); - throw new RuntimeException(e); - } - })); - } - - futures.forEach(future -> { - try { - future.get(); - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); - } - }); - } - } finally { - for (BatchExecutor batchExecutor : batchExecutors) { - if (batchExecutor != null) { - batchExecutor.close(); - } - } - } - } - - /** - * 批量同步 - * - * @param mappingConfig 配置集合 - * @param dmls 批量 DML - */ - public void sync(Map> mappingConfig, List dmls, Properties envProperties) { - sync(dmls, dml -> { - if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { - // DDL - columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable()); - return false; - } else { - // DML - String destination = StringUtils.trimToEmpty(dml.getDestination()); - String groupId = StringUtils.trimToEmpty(dml.getGroupId()); - String database = dml.getDatabase(); - String table = dml.getTable(); - Map configMap; - if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { - configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table); - } else { - configMap = mappingConfig.get(destination + "_" + database + "-" + table); - } - - if (configMap == null) { - return false; - } - - if (configMap.values().isEmpty()) { - return false; - } - - for (MappingConfig config : configMap.values()) { - if (config.getConcurrent()) { - List singleDmls = SingleDml.dml2SingleDmls(dml); - singleDmls.forEach(singleDml -> { - int hash = pkHash(config.getDbMapping(), singleDml.getData()); - SyncItem syncItem = new SyncItem(config, singleDml); - dmlsPartition[hash].add(syncItem); - }); - } else { - int hash = 0; - List singleDmls = SingleDml.dml2SingleDmls(dml); - singleDmls.forEach(singleDml -> { - SyncItem syncItem = new SyncItem(config, singleDml); - dmlsPartition[hash].add(syncItem); - }); - } - } - return true; - } - }); - } - - /** - * 单条 dml 同步 - * - * @param batchExecutor 批量事务执行器 - * @param config 对应配置对象 - * @param dml DML - */ - public void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { - if (config != null) { - try { - String type = dml.getType(); - if (type != null && type.equalsIgnoreCase("INSERT")) { - insert(batchExecutor, config, dml); - } else if (type != null && type.equalsIgnoreCase("UPDATE")) { - update(batchExecutor, config, dml); - } else if (type != null && type.equalsIgnoreCase("DELETE")) { - delete(batchExecutor, config, dml); - } else if (type != null && type.equalsIgnoreCase("TRUNCATE")) { - truncate(batchExecutor, config); - } - if (logger.isDebugEnabled()) { - logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); - } - } catch (SQLException e) { - throw new RuntimeException(e); - } - } - } - - /** - * 插入操作 - * - * @param config 配置项 - * @param dml DML数据 - */ - private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { - Map data = dml.getData(); - if (data == null || data.isEmpty()) { - return; - } - - DbMapping dbMapping = config.getDbMapping(); - - Map columnsMap = SyncUtil.getColumnsMap(dbMapping, data); - - StringBuilder insertSql = new StringBuilder(); - insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); - - columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); - int len = insertSql.length(); - insertSql.delete(len - 1, len).append(") VALUES ("); - int mapLen = columnsMap.size(); - for (int i = 0; i < mapLen; i++) { - insertSql.append("?,"); - } - len = insertSql.length(); - insertSql.delete(len - 1, len).append(")"); - - Map ctype = getTargetColumnType(batchExecutor.getConn(), config); - - List> values = new ArrayList<>(); - for (Map.Entry entry : columnsMap.entrySet()) { - String targetColumnName = entry.getKey(); - String srcColumnName = entry.getValue(); - if (srcColumnName == null) { - srcColumnName = Util.cleanColumn(targetColumnName); - } - - Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); - if (type == null) { - throw new RuntimeException("Target column: " + targetColumnName + " not matched"); - } - Object value = data.get(srcColumnName); - BatchExecutor.setValue(values, type, value); - } - - try { - batchExecutor.execute(insertSql.toString(), values); - } catch (SQLException e) { - if (skipDupException - && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001: 违反唯一约束条件"))) { - // ignore - // TODO 增加更多关系数据库的主键冲突的错误码 - } else { - throw e; - } - } - if (logger.isTraceEnabled()) { - logger.trace("Insert into target table, sql: {}", insertSql); - } - - } - - /** - * 更新操作 - * - * @param config 配置项 - * @param dml DML数据 - */ - private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { - Map data = dml.getData(); - if (data == null || data.isEmpty()) { - return; - } - - Map old = dml.getOld(); - if (old == null || old.isEmpty()) { - return; - } - - DbMapping dbMapping = config.getDbMapping(); - - Map columnsMap = SyncUtil.getColumnsMap(dbMapping, data); - - Map ctype = getTargetColumnType(batchExecutor.getConn(), config); - - StringBuilder updateSql = new StringBuilder(); - updateSql.append("UPDATE ").append(SyncUtil.getDbTableName(dbMapping)).append(" SET "); - List> values = new ArrayList<>(); - boolean hasMatched = false; - for (String srcColumnName : old.keySet()) { - List targetColumnNames = new ArrayList<>(); - columnsMap.forEach((targetColumn, srcColumn) -> { - if (srcColumnName.toLowerCase().equals(srcColumn.toLowerCase())) { - targetColumnNames.add(targetColumn); - } - }); - if (!targetColumnNames.isEmpty()) { - hasMatched = true; - for (String targetColumnName : targetColumnNames) { - updateSql.append(targetColumnName).append("=?, "); - Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); - if (type == null) { - throw new RuntimeException("Target column: " + targetColumnName + " not matched"); - } - BatchExecutor.setValue(values, type, data.get(srcColumnName)); - } - } - } - if (!hasMatched) { - logger.warn("Did not matched any columns to update "); - return; - } - int len = updateSql.length(); - updateSql.delete(len - 2, len).append(" WHERE "); - - // 拼接主键 - appendCondition(dbMapping, updateSql, ctype, values, data, old); - batchExecutor.execute(updateSql.toString(), values); - if (logger.isTraceEnabled()) { - logger.trace("Update target table, sql: {}", updateSql); - } - } - - /** - * 删除操作 - * - * @param config - * @param dml - */ - private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { - Map data = dml.getData(); - if (data == null || data.isEmpty()) { - return; - } - - DbMapping dbMapping = config.getDbMapping(); - - Map ctype = getTargetColumnType(batchExecutor.getConn(), config); - - StringBuilder sql = new StringBuilder(); - sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE "); - - List> values = new ArrayList<>(); - // 拼接主键 - appendCondition(dbMapping, sql, ctype, values, data); - batchExecutor.execute(sql.toString(), values); - if (logger.isTraceEnabled()) { - logger.trace("Delete from target table, sql: {}", sql); - } - } - - /** - * truncate操作 - * - * @param config - */ - private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException { - DbMapping dbMapping = config.getDbMapping(); - StringBuilder sql = new StringBuilder(); - sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping)); - batchExecutor.execute(sql.toString(), new ArrayList<>()); - if (logger.isTraceEnabled()) { - logger.trace("Truncate target table, sql: {}", sql); - } - } - - /** - * 获取目标字段类型 - * - * @param conn sql connection - * @param config 映射配置 - * @return 字段sqlType - */ - private Map getTargetColumnType(Connection conn, MappingConfig config) { - DbMapping dbMapping = config.getDbMapping(); - String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable(); - Map columnType = columnsTypeCache.get(cacheKey); - if (columnType == null) { - synchronized (RdbSyncService.class) { - columnType = columnsTypeCache.get(cacheKey); - if (columnType == null) { - columnType = new LinkedHashMap<>(); - final Map columnTypeTmp = columnType; - String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2"; - Util.sqlRS(conn, sql, rs -> { - try { - ResultSetMetaData rsd = rs.getMetaData(); - int columnCount = rsd.getColumnCount(); - for (int i = 1; i <= columnCount; i++) { - columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i)); - } - columnsTypeCache.put(cacheKey, columnTypeTmp); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - }); - } - } - } - return columnType; - } - - /** - * 拼接主键 where条件 - */ - private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map ctype, - List> values, Map d) { - appendCondition(dbMapping, sql, ctype, values, d, null); - } - - private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map ctype, - List> values, Map d, Map o) { - // 拼接主键 - for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { - String targetColumnName = entry.getKey(); - String srcColumnName = entry.getValue(); - if (srcColumnName == null) { - srcColumnName = Util.cleanColumn(targetColumnName); - } - sql.append(targetColumnName).append("=? AND "); - Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); - if (type == null) { - throw new RuntimeException("Target column: " + targetColumnName + " not matched"); - } - // 如果有修改主键的情况 - if (o != null && o.containsKey(srcColumnName)) { - BatchExecutor.setValue(values, type, o.get(srcColumnName)); - } else { - BatchExecutor.setValue(values, type, d.get(srcColumnName)); - } - } - int len = sql.length(); - sql.delete(len - 4, len); - } - - public static class SyncItem { - - private MappingConfig config; - private SingleDml singleDml; - - public SyncItem(MappingConfig config, SingleDml singleDml){ - this.config = config; - this.singleDml = singleDml; - } - } - - /** - * 取主键hash - */ - public int pkHash(DbMapping dbMapping, Map d) { - return pkHash(dbMapping, d, null); - } - - public int pkHash(DbMapping dbMapping, Map d, Map o) { - int hash = 0; - // 取主键 - for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { - String targetColumnName = entry.getKey(); - String srcColumnName = entry.getValue(); - if (srcColumnName == null) { - srcColumnName = Util.cleanColumn(targetColumnName); - } - Object value = null; - if (o != null && o.containsKey(srcColumnName)) { - value = o.get(srcColumnName); - } else if (d != null) { - value = d.get(srcColumnName); - } - if (value != null) { - hash += value.hashCode(); - } - } - hash = Math.abs(hash) % threads; - return Math.abs(hash); - } - - public void close() { - for (int i = 0; i < threads; i++) { - executorThreads[i].shutdown(); - } - } -} +package com.alibaba.otter.canal.client.adapter.rdb.service; + +import java.sql.Connection; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; + +import javax.sql.DataSource; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.serializer.SerializerFeature; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; +import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping; +import com.alibaba.otter.canal.client.adapter.rdb.support.BatchExecutor; +import com.alibaba.otter.canal.client.adapter.rdb.support.SingleDml; +import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; +import com.alibaba.otter.canal.client.adapter.support.Dml; +import com.alibaba.otter.canal.client.adapter.support.Util; + +/** + * RDB同步操作业务 + * + * @author rewerma 2018-11-7 下午06:45:49 + * @version 1.0.0 + */ +public class RdbSyncService { + + private static final Logger logger = LoggerFactory.getLogger(RdbSyncService.class); + + // 源库表字段类型缓存: instance.schema.table -> + private Map> columnsTypeCache; + + private int threads = 3; + private boolean skipDupException; + + private List[] dmlsPartition; + private BatchExecutor[] batchExecutors; + private ExecutorService[] executorThreads; + + public List[] getDmlsPartition() { + return dmlsPartition; + } + + public Map> getColumnsTypeCache() { + return columnsTypeCache; + } + + public RdbSyncService(DataSource dataSource, Integer threads, boolean skipDupException){ + this(dataSource, threads, new ConcurrentHashMap<>(), skipDupException); + } + + @SuppressWarnings("unchecked") + public RdbSyncService(DataSource dataSource, Integer threads, Map> columnsTypeCache, + boolean skipDupException){ + this.columnsTypeCache = columnsTypeCache; + this.skipDupException = skipDupException; + try { + if (threads != null) { + this.threads = threads; + } + this.dmlsPartition = new List[this.threads]; + this.batchExecutors = new BatchExecutor[this.threads]; + this.executorThreads = new ExecutorService[this.threads]; + for (int i = 0; i < this.threads; i++) { + dmlsPartition[i] = new ArrayList<>(); + batchExecutors[i] = new BatchExecutor(dataSource); + executorThreads[i] = Executors.newSingleThreadExecutor(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * 批量同步回调 + * + * @param dmls 批量 DML + * @param function 回调方法 + */ + public void sync(List dmls, Function function) { + try { + boolean toExecute = false; + for (Dml dml : dmls) { + if (!toExecute) { + toExecute = function.apply(dml); + } else { + function.apply(dml); + } + } + if (toExecute) { + List> futures = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + int j = i; + futures.add(executorThreads[i].submit(() -> { + try { + dmlsPartition[j] + .forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml)); + dmlsPartition[j].clear(); + batchExecutors[j].commit(); + return true; + } catch (Throwable e) { + batchExecutors[j].rollback(); + throw new RuntimeException(e); + } + })); + } + + futures.forEach(future -> { + try { + future.get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + }); + } + } finally { + for (BatchExecutor batchExecutor : batchExecutors) { + if (batchExecutor != null) { + batchExecutor.close(); + } + } + } + } + + /** + * 批量同步 + * + * @param mappingConfig 配置集合 + * @param dmls 批量 DML + */ + public void sync(Map> mappingConfig, List dmls, Properties envProperties) { + sync(dmls, dml -> { + if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) { + // DDL + columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable()); + return false; + } else { + // DML + String destination = StringUtils.trimToEmpty(dml.getDestination()); + String groupId = StringUtils.trimToEmpty(dml.getGroupId()); + String database = dml.getDatabase(); + String table = dml.getTable(); + Map configMap; + if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) { + configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table); + } else { + configMap = mappingConfig.get(destination + "_" + database + "-" + table); + } + + if (configMap == null) { + return false; + } + + if (configMap.values().isEmpty()) { + return false; + } + + for (MappingConfig config : configMap.values()) { + if (config.getConcurrent()) { + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + int hash = pkHash(config.getDbMapping(), singleDml.getData()); + SyncItem syncItem = new SyncItem(config, singleDml); + dmlsPartition[hash].add(syncItem); + }); + } else { + int hash = 0; + List singleDmls = SingleDml.dml2SingleDmls(dml); + singleDmls.forEach(singleDml -> { + SyncItem syncItem = new SyncItem(config, singleDml); + dmlsPartition[hash].add(syncItem); + }); + } + } + return true; + } + }); + } + + /** + * 单条 dml 同步 + * + * @param batchExecutor 批量事务执行器 + * @param config 对应配置对象 + * @param dml DML + */ + public void sync(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) { + if (config != null) { + try { + String type = dml.getType(); + if (type != null && type.equalsIgnoreCase("INSERT")) { + insert(batchExecutor, config, dml); + } else if (type != null && type.equalsIgnoreCase("UPDATE")) { + update(batchExecutor, config, dml); + } else if (type != null && type.equalsIgnoreCase("DELETE")) { + delete(batchExecutor, config, dml); + } else if (type != null && type.equalsIgnoreCase("TRUNCATE")) { + truncate(batchExecutor, config); + } + if (logger.isDebugEnabled()) { + logger.debug("DML: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue)); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + + /** + * 插入操作 + * + * @param config 配置项 + * @param dml DML数据 + */ + private void insert(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { + Map data = dml.getData(); + if (data == null || data.isEmpty()) { + return; + } + + DbMapping dbMapping = config.getDbMapping(); + + Map columnsMap = SyncUtil.getColumnsMap(dbMapping, data); + + StringBuilder insertSql = new StringBuilder(); + insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); + + columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); + int len = insertSql.length(); + insertSql.delete(len - 1, len).append(") VALUES ("); + int mapLen = columnsMap.size(); + for (int i = 0; i < mapLen; i++) { + insertSql.append("?,"); + } + len = insertSql.length(); + insertSql.delete(len - 1, len).append(")"); + + Map ctype = getTargetColumnType(batchExecutor.getConn(), config); + + List> values = new ArrayList<>(); + for (Map.Entry entry : columnsMap.entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = Util.cleanColumn(targetColumnName); + } + + Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); + if (type == null) { + throw new RuntimeException("Target column: " + targetColumnName + " not matched"); + } + Object value = data.get(srcColumnName); + BatchExecutor.setValue(values, type, value); + } + + try { + batchExecutor.execute(insertSql.toString(), values); + } catch (SQLException e) { + if (skipDupException + && (e.getMessage().contains("Duplicate entry") || e.getMessage().startsWith("ORA-00001: 违反唯一约束条件"))) { + // ignore + // TODO 增加更多关系数据库的主键冲突的错误码 + } else { + throw e; + } + } + if (logger.isTraceEnabled()) { + logger.trace("Insert into target table, sql: {}", insertSql); + } + + } + + /** + * 更新操作 + * + * @param config 配置项 + * @param dml DML数据 + */ + private void update(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { + Map data = dml.getData(); + if (data == null || data.isEmpty()) { + return; + } + + Map old = dml.getOld(); + if (old == null || old.isEmpty()) { + return; + } + + DbMapping dbMapping = config.getDbMapping(); + + Map columnsMap = SyncUtil.getColumnsMap(dbMapping, data); + + Map ctype = getTargetColumnType(batchExecutor.getConn(), config); + + StringBuilder updateSql = new StringBuilder(); + updateSql.append("UPDATE ").append(SyncUtil.getDbTableName(dbMapping)).append(" SET "); + List> values = new ArrayList<>(); + boolean hasMatched = false; + for (String srcColumnName : old.keySet()) { + List targetColumnNames = new ArrayList<>(); + columnsMap.forEach((targetColumn, srcColumn) -> { + if (srcColumnName.toLowerCase().equals(srcColumn.toLowerCase())) { + targetColumnNames.add(targetColumn); + } + }); + if (!targetColumnNames.isEmpty()) { + hasMatched = true; + for (String targetColumnName : targetColumnNames) { + updateSql.append(targetColumnName).append("=?, "); + Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); + if (type == null) { + throw new RuntimeException("Target column: " + targetColumnName + " not matched"); + } + BatchExecutor.setValue(values, type, data.get(srcColumnName)); + } + } + } + if (!hasMatched) { + logger.warn("Did not matched any columns to update "); + return; + } + int len = updateSql.length(); + updateSql.delete(len - 2, len).append(" WHERE "); + + // 拼接主键 + appendCondition(dbMapping, updateSql, ctype, values, data, old); + batchExecutor.execute(updateSql.toString(), values); + if (logger.isTraceEnabled()) { + logger.trace("Update target table, sql: {}", updateSql); + } + } + + /** + * 删除操作 + * + * @param config + * @param dml + */ + private void delete(BatchExecutor batchExecutor, MappingConfig config, SingleDml dml) throws SQLException { + Map data = dml.getData(); + if (data == null || data.isEmpty()) { + return; + } + + DbMapping dbMapping = config.getDbMapping(); + + Map ctype = getTargetColumnType(batchExecutor.getConn(), config); + + StringBuilder sql = new StringBuilder(); + sql.append("DELETE FROM ").append(SyncUtil.getDbTableName(dbMapping)).append(" WHERE "); + + List> values = new ArrayList<>(); + // 拼接主键 + appendCondition(dbMapping, sql, ctype, values, data); + batchExecutor.execute(sql.toString(), values); + if (logger.isTraceEnabled()) { + logger.trace("Delete from target table, sql: {}", sql); + } + } + + /** + * truncate操作 + * + * @param config + */ + private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException { + DbMapping dbMapping = config.getDbMapping(); + StringBuilder sql = new StringBuilder(); + sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping)); + batchExecutor.execute(sql.toString(), new ArrayList<>()); + if (logger.isTraceEnabled()) { + logger.trace("Truncate target table, sql: {}", sql); + } + } + + /** + * 获取目标字段类型 + * + * @param conn sql connection + * @param config 映射配置 + * @return 字段sqlType + */ + private Map getTargetColumnType(Connection conn, MappingConfig config) { + DbMapping dbMapping = config.getDbMapping(); + String cacheKey = config.getDestination() + "." + dbMapping.getDatabase() + "." + dbMapping.getTable(); + Map columnType = columnsTypeCache.get(cacheKey); + if (columnType == null) { + synchronized (RdbSyncService.class) { + columnType = columnsTypeCache.get(cacheKey); + if (columnType == null) { + columnType = new LinkedHashMap<>(); + final Map columnTypeTmp = columnType; + String sql = "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE 1=2"; + Util.sqlRS(conn, sql, rs -> { + try { + ResultSetMetaData rsd = rs.getMetaData(); + int columnCount = rsd.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + columnTypeTmp.put(rsd.getColumnName(i).toLowerCase(), rsd.getColumnType(i)); + } + columnsTypeCache.put(cacheKey, columnTypeTmp); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + }); + } + } + } + return columnType; + } + + /** + * 拼接主键 where条件 + */ + private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map ctype, + List> values, Map d) { + appendCondition(dbMapping, sql, ctype, values, d, null); + } + + private void appendCondition(MappingConfig.DbMapping dbMapping, StringBuilder sql, Map ctype, + List> values, Map d, Map o) { + // 拼接主键 + for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = Util.cleanColumn(targetColumnName); + } + sql.append(targetColumnName).append("=? AND "); + Integer type = ctype.get(Util.cleanColumn(targetColumnName).toLowerCase()); + if (type == null) { + throw new RuntimeException("Target column: " + targetColumnName + " not matched"); + } + // 如果有修改主键的情况 + if (o != null && o.containsKey(srcColumnName)) { + BatchExecutor.setValue(values, type, o.get(srcColumnName)); + } else { + BatchExecutor.setValue(values, type, d.get(srcColumnName)); + } + } + int len = sql.length(); + sql.delete(len - 4, len); + } + + public static class SyncItem { + + private MappingConfig config; + private SingleDml singleDml; + + public SyncItem(MappingConfig config, SingleDml singleDml){ + this.config = config; + this.singleDml = singleDml; + } + } + + /** + * 取主键hash + */ + public int pkHash(DbMapping dbMapping, Map d) { + return pkHash(dbMapping, d, null); + } + + public int pkHash(DbMapping dbMapping, Map d, Map o) { + int hash = 0; + // 取主键 + for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { + String targetColumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = Util.cleanColumn(targetColumnName); + } + Object value = null; + if (o != null && o.containsKey(srcColumnName)) { + value = o.get(srcColumnName); + } else if (d != null) { + value = d.get(srcColumnName); + } + if (value != null) { + hash += value.hashCode(); + } + } + hash = Math.abs(hash) % threads; + return Math.abs(hash); + } + + public void close() { + for (int i = 0; i < threads; i++) { + executorThreads[i].shutdown(); + } + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java index f3bae97d..199e04ab 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java @@ -1,326 +1,325 @@ -package com.alibaba.otter.canal.client.kafka; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import com.google.common.collect.MapMaker; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.serialization.StringDeserializer; - -import com.alibaba.fastjson.JSON; -import com.alibaba.otter.canal.client.CanalMQConnector; -import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; -import com.alibaba.otter.canal.protocol.FlatMessage; -import com.alibaba.otter.canal.protocol.Message; -import com.alibaba.otter.canal.protocol.exception.CanalClientException; -import com.google.common.collect.Lists; - -/** - * canal kafka 数据操作客户端 - * - * 

- * 注意点：
- * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
- *
- * - * @author machengyuan @ 2018-6-12 - * @version 1.1.1 - */ -public class KafkaCanalConnector implements CanalMQConnector { - - protected KafkaConsumer kafkaConsumer; - protected KafkaConsumer kafkaConsumer2; // 用于扁平message的数据消费 - protected String topic; - protected Integer partition; - protected Properties properties; - protected volatile boolean connected = false; - protected volatile boolean running = false; - protected boolean flatMessage; - - private Map currentOffsets = new ConcurrentHashMap<>(); - - public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize, - boolean flatMessage){ - this.topic = topic; - this.partition = partition; - this.flatMessage = flatMessage; - - properties = new Properties(); - properties.put("bootstrap.servers", servers); - properties.put("group.id", groupId); - properties.put("enable.auto.commit", false); - properties.put("auto.commit.interval.ms", "1000"); - properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读 - properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 - properties.put("session.timeout.ms", "30000"); // 默认为30秒 - if (batchSize == null) { - batchSize = 100; - } - properties.put("max.poll.records", batchSize.toString()); - properties.put("key.deserializer", StringDeserializer.class.getName()); - if (!flatMessage) { - properties.put("value.deserializer", MessageDeserializer.class.getName()); - } else { - properties.put("value.deserializer", StringDeserializer.class.getName()); - } - } - - /** - * 打开连接 - */ - @Override - public void connect() { - if (connected) { - return; - } - - connected = true; - if (kafkaConsumer == null && !flatMessage) { - kafkaConsumer = new KafkaConsumer(properties); - - } - if (kafkaConsumer2 == null && flatMessage) { - kafkaConsumer2 = new KafkaConsumer(properties); - } - } - - /** - * 关闭链接 - */ - @Override - public void disconnect() { - if (kafkaConsumer != null) { - kafkaConsumer.close(); - kafkaConsumer = null; - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.close(); - kafkaConsumer2 = null; - } - - connected = false; - } - - protected void waitClientRunning() { - running = true; - } - - @Override - public boolean checkValid() { - return true;// 默认都放过 - } - - /** - * 订阅topic - */ - @Override - public void subscribe() { - waitClientRunning(); - if (!running) { - return; - } - - if (partition == null) { - if (kafkaConsumer != null) { - kafkaConsumer.subscribe(Collections.singletonList(topic)); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.subscribe(Collections.singletonList(topic)); - } - } else { - TopicPartition topicPartition = new TopicPartition(topic, partition); - if (kafkaConsumer != null) { - kafkaConsumer.assign(Collections.singletonList(topicPartition)); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.assign(Collections.singletonList(topicPartition)); - } - } - } - - /** - * 取消订阅 - */ - @Override - public void unsubscribe() { - waitClientRunning(); - if (!running) { - return; - } - - if (kafkaConsumer != null) { - kafkaConsumer.unsubscribe(); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.unsubscribe(); - } - } - - @Override - public List getList(Long timeout, TimeUnit unit) throws CanalClientException { - waitClientRunning(); - if (!running) { - return Lists.newArrayList(); - } - - List messages = getListWithoutAck(timeout, unit); - if (messages != null && !messages.isEmpty()) { - this.ack(); - } - return messages; - } - - @Override - public List getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { - waitClientRunning(); - if (!running) { - return Lists.newArrayList(); - } - - ConsumerRecords records = kafkaConsumer.poll(unit.toMillis(timeout)); - - currentOffsets.clear(); - for (TopicPartition topicPartition : records.partitions()) { - currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition)); - } - - if (!records.isEmpty()) { - List messages = new ArrayList<>(); - for (ConsumerRecord record : records) { - messages.add(record.value()); - } - return messages; - } - return Lists.newArrayList(); - } - - @Override - public List getFlatList(Long timeout, TimeUnit unit) throws CanalClientException { - waitClientRunning(); - if (!running) { - return Lists.newArrayList(); - } - - List messages = getFlatListWithoutAck(timeout, unit); - if (messages != null && !messages.isEmpty()) { - this.ack(); - } - return messages; - } - - @Override - public List getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { - waitClientRunning(); - if (!running) { - return Lists.newArrayList(); - } - - ConsumerRecords records = kafkaConsumer2.poll(unit.toMillis(timeout)); - - currentOffsets.clear(); - for (TopicPartition topicPartition : records.partitions()) { - currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition)); - } - - if (!records.isEmpty()) { - List flatMessages = new ArrayList<>(); - for (ConsumerRecord record : records) { - String flatMessageJson = record.value(); - FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); - flatMessages.add(flatMessage); - } - - return flatMessages; - } - return Lists.newArrayList(); - } - - @Override - public void rollback() { - waitClientRunning(); - if (!running) { - return; - } - // 回滚所有分区 - if (kafkaConsumer != null) { - for (Map.Entry entry : currentOffsets.entrySet()) { - kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); - } - } - if (kafkaConsumer2 != null) { - for (Map.Entry entry : currentOffsets.entrySet()) { - kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); - } - } - } - - /** - * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败 - */ - @Override - public void ack() { - waitClientRunning(); - if (!running) { - return; - } - - if (kafkaConsumer != null) { - kafkaConsumer.commitSync(); - } - if (kafkaConsumer2 != null) { - kafkaConsumer2.commitSync(); - } - } - - @Override - public void subscribe(String filter) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - @Override - public Message get(int batchSize) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - @Override - public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - @Override - public Message getWithoutAck(int batchSize) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - @Override - public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - @Override - public void ack(long batchId) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - @Override - public void rollback(long batchId) throws CanalClientException { - throw new CanalClientException("mq not support this method"); - } - - /** - * 重新设置sessionTime - * - * @param timeout - * @param unit - */ - public void setSessionTimeout(Long timeout, TimeUnit unit) { - long t = unit.toMillis(timeout); - properties.put("request.timeout.ms", String.valueOf(t + 60000)); - properties.put("session.timeout.ms", String.valueOf(t)); - } - -} +package com.alibaba.otter.canal.client.kafka; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.StringDeserializer; + +import com.alibaba.fastjson.JSON; +import com.alibaba.otter.canal.client.CanalMQConnector; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.protocol.FlatMessage; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.common.collect.Lists; + +/** + * canal kafka 数据操作客户端 + * + * 
+ * 注意点：
+ * 1. 相比于canal {@linkplain SimpleCanalConnector}, 这里get和ack操作不能有并发, 必须是一个线程执行get后，内存里执行完毕ack后再取下一个get
+ *
+ * + * @author machengyuan @ 2018-6-12 + * @version 1.1.1 + */ +public class KafkaCanalConnector implements CanalMQConnector { + + protected KafkaConsumer kafkaConsumer; + protected KafkaConsumer kafkaConsumer2; // 用于扁平message的数据消费 + protected String topic; + protected Integer partition; + protected Properties properties; + protected volatile boolean connected = false; + protected volatile boolean running = false; + protected boolean flatMessage; + + private Map currentOffsets = new ConcurrentHashMap<>(); + + public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId, Integer batchSize, + boolean flatMessage){ + this.topic = topic; + this.partition = partition; + this.flatMessage = flatMessage; + + properties = new Properties(); + properties.put("bootstrap.servers", servers); + properties.put("group.id", groupId); + properties.put("enable.auto.commit", false); + properties.put("auto.commit.interval.ms", "1000"); + properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读 + properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置 + properties.put("session.timeout.ms", "30000"); // 默认为30秒 + if (batchSize == null) { + batchSize = 100; + } + properties.put("max.poll.records", batchSize.toString()); + properties.put("key.deserializer", StringDeserializer.class.getName()); + if (!flatMessage) { + properties.put("value.deserializer", MessageDeserializer.class.getName()); + } else { + properties.put("value.deserializer", StringDeserializer.class.getName()); + } + } + + /** + * 打开连接 + */ + @Override + public void connect() { + if (connected) { + return; + } + + connected = true; + if (kafkaConsumer == null && !flatMessage) { + kafkaConsumer = new KafkaConsumer(properties); + + } + if (kafkaConsumer2 == null && flatMessage) { + kafkaConsumer2 = new KafkaConsumer(properties); + } + } + + /** + * 关闭链接 + */ + @Override + public void disconnect() { + if (kafkaConsumer != null) { + kafkaConsumer.close(); + kafkaConsumer = null; + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.close(); + kafkaConsumer2 = null; + } + + connected = false; + } + + protected void waitClientRunning() { + running = true; + } + + @Override + public boolean checkValid() { + return true;// 默认都放过 + } + + /** + * 订阅topic + */ + @Override + public void subscribe() { + waitClientRunning(); + if (!running) { + return; + } + + if (partition == null) { + if (kafkaConsumer != null) { + kafkaConsumer.subscribe(Collections.singletonList(topic)); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.subscribe(Collections.singletonList(topic)); + } + } else { + TopicPartition topicPartition = new TopicPartition(topic, partition); + if (kafkaConsumer != null) { + kafkaConsumer.assign(Collections.singletonList(topicPartition)); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.assign(Collections.singletonList(topicPartition)); + } + } + } + + /** + * 取消订阅 + */ + @Override + public void unsubscribe() { + waitClientRunning(); + if (!running) { + return; + } + + if (kafkaConsumer != null) { + kafkaConsumer.unsubscribe(); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.unsubscribe(); + } + } + + @Override + public List getList(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + List messages = getListWithoutAck(timeout, unit); + if (messages != null && !messages.isEmpty()) { + this.ack(); + } + return messages; + } + + @Override + public List getListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + ConsumerRecords records = kafkaConsumer.poll(unit.toMillis(timeout)); + + currentOffsets.clear(); + for (TopicPartition topicPartition : records.partitions()) { + currentOffsets.put(topicPartition.partition(), kafkaConsumer.position(topicPartition)); + } + + if (!records.isEmpty()) { + List messages = new ArrayList<>(); + for (ConsumerRecord record : records) { + messages.add(record.value()); + } + return messages; + } + return Lists.newArrayList(); + } + + @Override + public List getFlatList(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + List messages = getFlatListWithoutAck(timeout, unit); + if (messages != null && !messages.isEmpty()) { + this.ack(); + } + return messages; + } + + @Override + public List getFlatListWithoutAck(Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + if (!running) { + return Lists.newArrayList(); + } + + ConsumerRecords records = kafkaConsumer2.poll(unit.toMillis(timeout)); + + currentOffsets.clear(); + for (TopicPartition topicPartition : records.partitions()) { + currentOffsets.put(topicPartition.partition(), kafkaConsumer2.position(topicPartition)); + } + + if (!records.isEmpty()) { + List flatMessages = new ArrayList<>(); + for (ConsumerRecord record : records) { + String flatMessageJson = record.value(); + FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class); + flatMessages.add(flatMessage); + } + + return flatMessages; + } + return Lists.newArrayList(); + } + + @Override + public void rollback() { + waitClientRunning(); + if (!running) { + return; + } + // 回滚所有分区 + if (kafkaConsumer != null) { + for (Map.Entry entry : currentOffsets.entrySet()) { + kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); + } + } + if (kafkaConsumer2 != null) { + for (Map.Entry entry : currentOffsets.entrySet()) { + kafkaConsumer2.seek(new TopicPartition(topic, entry.getKey()), entry.getValue() - 1); + } + } + } + + /** + * 提交offset，如果超过 session.timeout.ms 设置的时间没有ack则会抛出异常，ack失败 + */ + @Override + public void ack() { + waitClientRunning(); + if (!running) { + return; + } + + if (kafkaConsumer != null) { + kafkaConsumer.commitSync(); + } + if (kafkaConsumer2 != null) { + kafkaConsumer2.commitSync(); + } + } + + @Override + public void subscribe(String filter) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message get(int batchSize) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message getWithoutAck(int batchSize) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public void ack(long batchId) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + @Override + public void rollback(long batchId) throws CanalClientException { + throw new CanalClientException("mq not support this method"); + } + + /** + * 重新设置sessionTime + * + * @param timeout + * @param unit + */ + public void setSessionTimeout(Long timeout, TimeUnit unit) { + long t = unit.toMillis(timeout); + properties.put("request.timeout.ms", String.valueOf(t + 60000)); + properties.put("session.timeout.ms", String.valueOf(t)); + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java index 3661ec9e..62898a1d 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaFlatMessage.java @@ -1,32 +1,35 @@ -package com.alibaba.otter.canal.client.kafka.protocol; - -import com.alibaba.otter.canal.protocol.FlatMessage; -import org.springframework.beans.BeanUtils; - -/** - * 消息对象（Kafka） - * - * @Author panjianping - * @Email ipanjianping@qq.com - * @Date 2018/12/17 - */ -public class KafkaFlatMessage extends FlatMessage { - /** - * Kafka 消息 offset - */ - private long offset; - - public KafkaFlatMessage(FlatMessage message, long offset) { - super(message.getId()); - BeanUtils.copyProperties(message, this); - this.offset = offset; - } - - public long getOffset() { - return offset; - } - - public void setOffset(long offset) { - this.offset = offset; - } -} +package com.alibaba.otter.canal.client.kafka.protocol; + +import com.alibaba.otter.canal.protocol.FlatMessage; +import org.springframework.beans.BeanUtils; + +/** + * 消息对象（Kafka） + * + * @Author panjianping + * @Email ipanjianping@qq.com + * @Date 2018/12/17 + */ +public class KafkaFlatMessage extends FlatMessage { + + private static final long serialVersionUID = 5748024400508080710L; + + /** + * Kafka 消息 offset + */ + private long offset; + + public KafkaFlatMessage(FlatMessage message, long offset){ + super(message.getId()); + BeanUtils.copyProperties(message, this); + this.offset = offset; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java index f2f55a71..4cfafc69 100644 --- a/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java +++ b/client/src/main/java/com/alibaba/otter/canal/client/kafka/protocol/KafkaMessage.java @@ -1,33 +1,35 @@ -package com.alibaba.otter.canal.client.kafka.protocol; - -import com.alibaba.otter.canal.protocol.Message; -import org.springframework.beans.BeanUtils; - -/** - * 消息对象（Kafka） - * - * @Author panjianping - * @Email ipanjianping@qq.com - * @Date 2018/12/17 - */ -public class KafkaMessage extends Message { - /** - * Kafka 消息 offset - */ - private long offset; - - public KafkaMessage(Message message, long offset) { - super(message.getId()); - BeanUtils.copyProperties(message, this); - this.offset = offset; - } - - - public long getOffset() { - return offset; - } - - public void setOffset(long offset) { - this.offset = offset; - } -} +package com.alibaba.otter.canal.client.kafka.protocol; + +import com.alibaba.otter.canal.protocol.Message; +import org.springframework.beans.BeanUtils; + +/** + * 消息对象（Kafka） + * + * @Author panjianping + * @Email ipanjianping@qq.com + * @Date 2018/12/17 + */ +public class KafkaMessage extends Message { + + private static final long serialVersionUID = -293120358490119447L; + + /** + * Kafka 消息 offset + */ + private long offset; + + public KafkaMessage(Message message, long offset){ + super(message.getId()); + BeanUtils.copyProperties(message, this); + this.offset = offset; + } + + public long getOffset() { + return offset; + } + + public void setOffset(long offset) { + this.offset = offset; + } +} From 947cf130ab6a00741c5803b6d9c9cefadacc31d7 Mon Sep 17 00:00:00 2001 From: mcy Date: Wed, 13 Feb 2019 15:50:11 +0800 Subject: [PATCH 09/10] fixed #1504 --- .../client/adapter/es/support/ESTemplate.java | 1052 ++++++++--------- 1 file changed, 526 insertions(+), 526 deletions(-) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java index 9be2d801..e6442fa1 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java @@ -1,526 +1,526 @@ -package com.alibaba.otter.canal.client.adapter.es.support; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -import javax.sql.DataSource; - -import com.alibaba.fastjson.JSON; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.UpdateByQueryAction; -import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.SearchHit; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.CollectionUtils; - -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; -import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; -import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; - -/** - * ES 操作模板 - * - * @author rewerma 2018-11-01 - * @version 1.0.0 - */ -public class ESTemplate { - - private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); - - private static final int MAX_BATCH_SIZE = 1000; - - private TransportClient transportClient; - - public ESTemplate(TransportClient transportClient){ - this.transportClient = transportClient; - } - - /** - * 插入数据 - * - * @param mapping - * @param pkVal - * @param esFieldData - * @return - */ - public boolean insert(ESMapping mapping, Object pkVal, Map esFieldData) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - if (mapping.get_id() != null) { - bulkRequestBuilder - .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) - .setSource(esFieldData)); - } else { - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); - for (SearchHit hit : response.getHits()) { - bulkRequestBuilder - .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); - } - bulkRequestBuilder - .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData)); - } - return commitBulkRequest(bulkRequestBuilder); - } - - /** - * 根据主键更新数据 - * - * @param mapping - * @param pkVal - * @param esFieldData - * @return - */ - public boolean update(ESMapping mapping, Object pkVal, Map esFieldData) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData); - return commitBulkRequest(bulkRequestBuilder); - } - - public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, - Map esFieldData) { - if (mapping.get_id() != null) { - bulkRequestBuilder - .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) - .setDoc(esFieldData)); - } else { - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); - for (SearchHit hit : response.getHits()) { - bulkRequestBuilder - .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) - .setDoc(esFieldData)); - } - } - } - - /** - * update by query - * - * @param config - * @param paramsTmp - * @param esFieldData - * @return - */ - public boolean updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { - if (paramsTmp.isEmpty()) { - return false; - } - ESMapping mapping = config.getEsMapping(); - BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); - paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); - - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setSize(0) - .setQuery(queryBuilder) - .get(); - long count = response.getHits().getTotalHits(); - // 如果更新量大于Max, 查询sql批量更新 - if (count > MAX_BATCH_SIZE) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - - DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); - // 查询sql更新 - StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); - paramsTmp.forEach( - (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND ")); - int len = sql.length(); - sql.delete(len - 4, len); - ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { - int exeCount = 1; - try { - BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder; - while (rs.next()) { - Object idVal = getIdValFromRS(mapping, rs); - append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); - - if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { - commitBulkRequest(bulkRequestBuilderTmp); - bulkRequestBuilderTmp = transportClient.prepareBulk(); - } - exeCount++; - } - - if (bulkRequestBuilder.numberOfActions() > 0) { - commitBulkRequest(bulkRequestBuilderTmp); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - return 0; - }); - return true; - } else { - return updateByQuery(mapping, queryBuilder, esFieldData, 1); - } - } - - private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, - int counter) { - if (CollectionUtils.isEmpty(esFieldData)) { - return true; - } - - StringBuilder sb = new StringBuilder(); - esFieldData.forEach((key, value) -> { - if (value instanceof Map) { - Map mapValue = (Map) value; - if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) { - sb.append("ctx._source") - .append("['") - .append(key) - .append("']") - .append(" = [") - .append(mapValue.get("lon")) - .append(", ") - .append(mapValue.get("lat")) - .append("];"); - } else { - sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); - sb.append(JSON.toJSONString(value)); - sb.append(";"); - } - } else if (value instanceof List) { - sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); - sb.append(JSON.toJSONString(value)); - sb.append(";"); - } else if (value instanceof String) { - sb.append("ctx._source") - .append("['") - .append(key) - .append("']") - .append(" = '") - .append(value) - .append("';"); - } else { - sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";"); - } - }); - String scriptLine = sb.toString(); - if (logger.isTraceEnabled()) { - logger.trace(scriptLine); - } - - UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); - updateByQuery.source(mapping.get_index()) - .abortOnVersionConflict(false) - .filter(queryBuilder) - .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); - - BulkByScrollResponse response = updateByQuery.get(); - if (logger.isTraceEnabled()) { - logger.trace("updateByQuery response: {}", response.getStatus()); - } - if (!CollectionUtils.isEmpty(response.getSearchFailures())) { - logger.error("script update_for_search has search error: " + response.getBulkFailures()); - return false; - } - - if (!CollectionUtils.isEmpty(response.getBulkFailures())) { - logger.error("script update_for_search has update error: " + response.getBulkFailures()); - return false; - } - - if (response.getStatus().getVersionConflicts() > 0) { - if (counter >= 3) { - logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter); - return false; - } - logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行..."); - try { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) { - // ignore - } - return updateByQuery(mapping, queryBuilder, esFieldData, ++counter); - } - - return true; - } - - /** - * 通过主键删除数据 - * - * @param mapping - * @param pkVal - * @return - */ - public boolean delete(ESMapping mapping, Object pkVal) { - BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); - if (mapping.get_id() != null) { - bulkRequestBuilder - .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); - } else { - SearchResponse response = transportClient.prepareSearch(mapping.get_index()) - .setTypes(mapping.get_type()) - .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) - .setSize(MAX_BATCH_SIZE) - .get(); - for (SearchHit hit : response.getHits()) { - bulkRequestBuilder - .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); - } - } - return commitBulkRequest(bulkRequestBuilder); - } - - /** - * 批量提交 - * - * @param bulkRequestBuilder - * @return - */ - private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { - if (bulkRequestBuilder.numberOfActions() > 0) { - BulkResponse response = bulkRequestBuilder.execute().actionGet(); - if (response.hasFailures()) { - for (BulkItemResponse itemResponse : response.getItems()) { - if (!itemResponse.isFailed()) { - continue; - } - - if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { - logger.warn(itemResponse.getFailureMessage()); - } else { - logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); - } - } - } - - return !response.hasFailures(); - } - return true; - } - - public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, - String columnName) throws SQLException { - String esType = getEsType(mapping, fieldName); - - Object value = resultSet.getObject(columnName); - if (value instanceof Boolean) { - if (!"boolean".equals(esType)) { - value = resultSet.getByte(columnName); - } - } - - // 如果是对象类型 - if (mapping.getObjFields().containsKey(fieldName)) { - return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); - } else { - return ESSyncUtil.typeConvert(value, esType); - } - } - - public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, - Map esFieldData) throws SQLException { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = value; - } - - if (!fieldItem.getFieldName().equals(mapping.get_id()) - && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), value); - } - } - return resultIdVal; - } - - public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = value; - break; - } - } - return resultIdVal; - } - - public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, - Map esFieldData) throws SQLException { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); - } - - for (ColumnItem columnItem : fieldItem.getColumnItems()) { - if (dmlOld.containsKey(columnItem.getColumnName()) - && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), - getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); - break; - } - } - } - return resultIdVal; - } - - public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { - String esType = getEsType(mapping, fieldName); - Object value = dmlData.get(columnName); - if (value instanceof Byte) { - if ("boolean".equals(esType)) { - value = ((Byte) value).intValue() != 0; - } - } - - // 如果是对象类型 - if (mapping.getObjFields().containsKey(fieldName)) { - return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); - } else { - return ESSyncUtil.typeConvert(value, esType); - } - } - - /** - * 将dml的data转换为es的data - * - * @param mapping 配置mapping - * @param dmlData dml data - * @param esFieldData es data - * @return 返回 id 值 - */ - public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, - Map esFieldData) { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); - Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = value; - } - - if (!fieldItem.getFieldName().equals(mapping.get_id()) - && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), value); - } - } - return resultIdVal; - } - - /** - * 将dml的data, old转换为es的data - * - * @param mapping 配置mapping - * @param dmlData dml data - * @param esFieldData es data - * @return 返回 id 值 - */ - public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, - Map esFieldData) { - SchemaItem schemaItem = mapping.getSchemaItem(); - String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); - Object resultIdVal = null; - for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { - String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); - - if (fieldItem.getFieldName().equals(idFieldName)) { - resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); - } - - if (dmlOld.get(columnName) != null && !mapping.getSkips().contains(fieldItem.getFieldName())) { - esFieldData.put(fieldItem.getFieldName(), - getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); - } - } - return resultIdVal; - } - - /** - * es 字段类型本地缓存 - */ - private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); - - /** - * 获取es mapping中的属性类型 - * - * @param mapping mapping配置 - * @param fieldName 属性名 - * @return 类型 - */ - @SuppressWarnings("unchecked") - private String getEsType(ESMapping mapping, String fieldName) { - String key = mapping.get_index() + "-" + mapping.get_type(); - Map fieldType = esFieldTypes.get(key); - if (fieldType == null) { - ImmutableOpenMap mappings; - try { - mappings = transportClient.admin() - .cluster() - .prepareState() - .execute() - .actionGet() - .getState() - .getMetaData() - .getIndices() - .get(mapping.get_index()) - .getMappings(); - } catch (NullPointerException e) { - throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); - } - MappingMetaData mappingMetaData = mappings.get(mapping.get_type()); - if (mappingMetaData == null) { - throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); - } - - fieldType = new LinkedHashMap<>(); - - Map sourceMap = mappingMetaData.getSourceAsMap(); - Map esMapping = (Map) sourceMap.get("properties"); - for (Map.Entry entry : esMapping.entrySet()) { - Map value = (Map) entry.getValue(); - if (value.containsKey("properties")) { - fieldType.put(entry.getKey(), "object"); - } else { - fieldType.put(entry.getKey(), (String) value.get("type")); - } - } - esFieldTypes.put(key, fieldType); - } - - return fieldType.get(fieldName); - } -} +package com.alibaba.otter.canal.client.adapter.es.support; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import javax.sql.DataSource; + +import com.alibaba.fastjson.JSON; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.UpdateByQueryAction; +import org.elasticsearch.index.reindex.UpdateByQueryRequestBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.SearchHit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig; +import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem; +import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem; +import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; + +/** + * ES 操作模板 + * + * @author rewerma 2018-11-01 + * @version 1.0.0 + */ +public class ESTemplate { + + private static final Logger logger = LoggerFactory.getLogger(ESTemplate.class); + + private static final int MAX_BATCH_SIZE = 1000; + + private TransportClient transportClient; + + public ESTemplate(TransportClient transportClient){ + this.transportClient = transportClient; + } + + /** + * 插入数据 + * + * @param mapping + * @param pkVal + * @param esFieldData + * @return + */ + public boolean insert(ESMapping mapping, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setSource(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + bulkRequestBuilder + .add(transportClient.prepareIndex(mapping.get_index(), mapping.get_type()).setSource(esFieldData)); + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 根据主键更新数据 + * + * @param mapping + * @param pkVal + * @param esFieldData + * @return + */ + public boolean update(ESMapping mapping, Object pkVal, Map esFieldData) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + append4Update(bulkRequestBuilder, mapping, pkVal, esFieldData); + return commitBulkRequest(bulkRequestBuilder); + } + + public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, + Map esFieldData) { + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) + .setDoc(esFieldData)); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) + .setDoc(esFieldData)); + } + } + } + + /** + * update by query + * + * @param config + * @param paramsTmp + * @param esFieldData + * @return + */ + public boolean updateByQuery(ESSyncConfig config, Map paramsTmp, Map esFieldData) { + if (paramsTmp.isEmpty()) { + return false; + } + ESMapping mapping = config.getEsMapping(); + BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value))); + + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setSize(0) + .setQuery(queryBuilder) + .get(); + long count = response.getHits().getTotalHits(); + // 如果更新量大于Max, 查询sql批量更新 + if (count > MAX_BATCH_SIZE) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + + DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey()); + // 查询sql更新 + StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE "); + paramsTmp.forEach( + (fieldName, value) -> sql.append("_v.").append(fieldName).append("=").append(value).append(" AND ")); + int len = sql.length(); + sql.delete(len - 4, len); + ESSyncUtil.sqlRS(ds, sql.toString(), rs -> { + int exeCount = 1; + try { + BulkRequestBuilder bulkRequestBuilderTmp = bulkRequestBuilder; + while (rs.next()) { + Object idVal = getIdValFromRS(mapping, rs); + append4Update(bulkRequestBuilderTmp, mapping, idVal, esFieldData); + + if (exeCount % mapping.getCommitBatch() == 0 && bulkRequestBuilderTmp.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + bulkRequestBuilderTmp = transportClient.prepareBulk(); + } + exeCount++; + } + + if (bulkRequestBuilder.numberOfActions() > 0) { + commitBulkRequest(bulkRequestBuilderTmp); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + return 0; + }); + return true; + } else { + return updateByQuery(mapping, queryBuilder, esFieldData, 1); + } + } + + private boolean updateByQuery(ESMapping mapping, QueryBuilder queryBuilder, Map esFieldData, + int counter) { + if (CollectionUtils.isEmpty(esFieldData)) { + return true; + } + + StringBuilder sb = new StringBuilder(); + esFieldData.forEach((key, value) -> { + if (value instanceof Map) { + Map mapValue = (Map) value; + if (mapValue.containsKey("lon") && mapValue.containsKey("lat") && mapValue.size() == 2) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = [") + .append(mapValue.get("lon")) + .append(", ") + .append(mapValue.get("lat")) + .append("];"); + } else { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); + } + } else if (value instanceof List) { + sb.append("ctx._source").append("[\"").append(key).append("\"]").append(" = "); + sb.append(JSON.toJSONString(value)); + sb.append(";"); + } else if (value instanceof String) { + sb.append("ctx._source") + .append("['") + .append(key) + .append("']") + .append(" = '") + .append(value) + .append("';"); + } else { + sb.append("ctx._source").append("['").append(key).append("']").append(" = ").append(value).append(";"); + } + }); + String scriptLine = sb.toString(); + if (logger.isTraceEnabled()) { + logger.trace(scriptLine); + } + + UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(transportClient); + updateByQuery.source(mapping.get_index()) + .abortOnVersionConflict(false) + .filter(queryBuilder) + .script(new Script(ScriptType.INLINE, "painless", scriptLine, Collections.emptyMap())); + + BulkByScrollResponse response = updateByQuery.get(); + if (logger.isTraceEnabled()) { + logger.trace("updateByQuery response: {}", response.getStatus()); + } + if (!CollectionUtils.isEmpty(response.getSearchFailures())) { + logger.error("script update_for_search has search error: " + response.getBulkFailures()); + return false; + } + + if (!CollectionUtils.isEmpty(response.getBulkFailures())) { + logger.error("script update_for_search has update error: " + response.getBulkFailures()); + return false; + } + + if (response.getStatus().getVersionConflicts() > 0) { + if (counter >= 3) { + logger.error("第 {} 次执行updateByQuery, 依旧存在分片版本冲突，不再继续重试。", counter); + return false; + } + logger.warn("本次updateByQuery存在分片版本冲突，准备重新执行..."); + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + // ignore + } + return updateByQuery(mapping, queryBuilder, esFieldData, ++counter); + } + + return true; + } + + /** + * 通过主键删除数据 + * + * @param mapping + * @param pkVal + * @return + */ + public boolean delete(ESMapping mapping, Object pkVal) { + BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk(); + if (mapping.get_id() != null) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), pkVal.toString())); + } else { + SearchResponse response = transportClient.prepareSearch(mapping.get_index()) + .setTypes(mapping.get_type()) + .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) + .setSize(MAX_BATCH_SIZE) + .get(); + for (SearchHit hit : response.getHits()) { + bulkRequestBuilder + .add(transportClient.prepareDelete(mapping.get_index(), mapping.get_type(), hit.getId())); + } + } + return commitBulkRequest(bulkRequestBuilder); + } + + /** + * 批量提交 + * + * @param bulkRequestBuilder + * @return + */ + private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { + if (bulkRequestBuilder.numberOfActions() > 0) { + BulkResponse response = bulkRequestBuilder.execute().actionGet(); + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (!itemResponse.isFailed()) { + continue; + } + + if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { + logger.warn(itemResponse.getFailureMessage()); + } else { + logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); + } + } + } + + return !response.hasFailures(); + } + return true; + } + + public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName, + String columnName) throws SQLException { + String esType = getEsType(mapping, fieldName); + + Object value = resultSet.getObject(columnName); + if (value instanceof Boolean) { + if (!"boolean".equals(esType)) { + value = resultSet.getByte(columnName); + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); + } + } + return resultIdVal; + } + + public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + break; + } + } + return resultIdVal; + } + + public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map dmlOld, + Map esFieldData) throws SQLException { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()); + } + + for (ColumnItem columnItem : fieldItem.getColumnItems()) { + if (dmlOld.containsKey(columnItem.getColumnName()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName())); + break; + } + } + } + return resultIdVal; + } + + public Object getValFromData(ESMapping mapping, Map dmlData, String fieldName, String columnName) { + String esType = getEsType(mapping, fieldName); + Object value = dmlData.get(columnName); + if (value instanceof Byte) { + if ("boolean".equals(esType)) { + value = ((Byte) value).intValue() != 0; + } + } + + // 如果是对象类型 + if (mapping.getObjFields().containsKey(fieldName)) { + return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName)); + } else { + return ESSyncUtil.typeConvert(value, esType); + } + } + + /** + * 将dml的data转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = value; + } + + if (!fieldItem.getFieldName().equals(mapping.get_id()) + && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), value); + } + } + return resultIdVal; + } + + /** + * 将dml的data, old转换为es的data + * + * @param mapping 配置mapping + * @param dmlData dml data + * @param esFieldData es data + * @return 返回 id 值 + */ + public Object getESDataFromDmlData(ESMapping mapping, Map dmlData, Map dmlOld, + Map esFieldData) { + SchemaItem schemaItem = mapping.getSchemaItem(); + String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id(); + Object resultIdVal = null; + for (FieldItem fieldItem : schemaItem.getSelectFields().values()) { + String columnName = fieldItem.getColumnItems().iterator().next().getColumnName(); + + if (fieldItem.getFieldName().equals(idFieldName)) { + resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName); + } + + if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) { + esFieldData.put(fieldItem.getFieldName(), + getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName)); + } + } + return resultIdVal; + } + + /** + * es 字段类型本地缓存 + */ + private static ConcurrentMap> esFieldTypes = new ConcurrentHashMap<>(); + + /** + * 获取es mapping中的属性类型 + * + * @param mapping mapping配置 + * @param fieldName 属性名 + * @return 类型 + */ + @SuppressWarnings("unchecked") + private String getEsType(ESMapping mapping, String fieldName) { + String key = mapping.get_index() + "-" + mapping.get_type(); + Map fieldType = esFieldTypes.get(key); + if (fieldType == null) { + ImmutableOpenMap mappings; + try { + mappings = transportClient.admin() + .cluster() + .prepareState() + .execute() + .actionGet() + .getState() + .getMetaData() + .getIndices() + .get(mapping.get_index()) + .getMappings(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + MappingMetaData mappingMetaData = mappings.get(mapping.get_type()); + if (mappingMetaData == null) { + throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index()); + } + + fieldType = new LinkedHashMap<>(); + + Map sourceMap = mappingMetaData.getSourceAsMap(); + Map esMapping = (Map) sourceMap.get("properties"); + for (Map.Entry entry : esMapping.entrySet()) { + Map value = (Map) entry.getValue(); + if (value.containsKey("properties")) { + fieldType.put(entry.getKey(), "object"); + } else { + fieldType.put(entry.getKey(), (String) value.get("type")); + } + } + esFieldTypes.put(key, fieldType); + } + + return fieldType.get(fieldName); + } +} From 05cc627b5da499fb43784259cd8bf4c2ac081468 Mon Sep 17 00:00:00 2001 From: mcy Date: Fri, 15 Feb 2019 11:39:26 +0800 Subject: [PATCH 10/10] fix rdb etl bug --- .../adapter/rdb/service/RdbEtlService.java | 190 +++++++++--------- 1 file changed, 100 insertions(+), 90 deletions(-) diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java index d1d19d3f..fbb188da 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/service/RdbEtlService.java @@ -13,12 +13,12 @@ import java.util.concurrent.atomic.AtomicLong; import javax.sql.DataSource; -import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig; import com.alibaba.otter.canal.client.adapter.rdb.config.MappingConfig.DbMapping; +import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; import com.alibaba.otter.canal.client.adapter.support.EtlResult; import com.alibaba.otter.canal.client.adapter.support.Util; import com.google.common.base.Joiner; @@ -36,7 +36,8 @@ public class RdbEtlService { /** * 导入数据 */ - public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, List params) { + public static EtlResult importData(DataSource srcDS, DataSource targetDS, MappingConfig config, + List params) { EtlResult etlResult = new EtlResult(); AtomicLong successCount = new AtomicLong(); List errMsg = new ArrayList<>(); @@ -53,8 +54,8 @@ public class RdbEtlService { long start = System.currentTimeMillis(); // 拼接sql - StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "." - + dbMapping.getTable()); + StringBuilder sql = new StringBuilder( + "SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable()); // 拼接条件 appendCondition(params, dbMapping, srcDS, sql); @@ -91,12 +92,8 @@ public class RdbEtlService { } else { sqlFinal = sql + " LIMIT " + offset + "," + cnt; } - Future future = executor.submit(() -> executeSqlImport(srcDS, - targetDS, - sqlFinal, - dbMapping, - successCount, - errMsg)); + Future future = executor + .submit(() -> executeSqlImport(srcDS, targetDS, sqlFinal, dbMapping, successCount, errMsg)); futures.add(future); } @@ -109,11 +106,11 @@ public class RdbEtlService { executeSqlImport(srcDS, targetDS, sql.toString(), dbMapping, successCount, errMsg); } - logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 - + "s!"); + logger.info( + dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!"); - etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() - + " 条"); + etlResult + .setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条"); } catch (Exception e) { logger.error(e.getMessage(), e); errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage()); @@ -127,8 +124,8 @@ public class RdbEtlService { return etlResult; } - private static void appendCondition(List params, DbMapping dbMapping, DataSource ds, StringBuilder sql) - throws SQLException { + private static void appendCondition(List params, DbMapping dbMapping, DataSource ds, + StringBuilder sql) throws SQLException { if (params != null && params.size() == 1 && dbMapping.getEtlCondition() == null) { AtomicBoolean stExists = new AtomicBoolean(false); // 验证是否有SYS_TIME字段 @@ -145,9 +142,9 @@ public class RdbEtlService { } } catch (Exception e) { // ignore - } - return null; - } ); + } + return null; + }); if (stExists.get()) { sql.append(" WHERE SYS_TIME >= '").append(params.get(0)).append("' "); } @@ -168,13 +165,12 @@ public class RdbEtlService { private static boolean executeSqlImport(DataSource srcDS, DataSource targetDS, String sql, DbMapping dbMapping, AtomicLong successCount, List errMsg) { try { - Util.sqlRS(srcDS, sql, rs -> { - int idx = 1; + Map columnsMap = new LinkedHashMap<>(); + Map columnType = new LinkedHashMap<>(); + Util.sqlRS(targetDS, "SELECT * FROM " + SyncUtil.getDbTableName(dbMapping) + " LIMIT 1 ", rs -> { try { - boolean completed = false; - Map columnType = new LinkedHashMap<>(); ResultSetMetaData rsd = rs.getMetaData(); int columnCount = rsd.getColumnCount(); List columns = new ArrayList<>(); @@ -183,91 +179,105 @@ public class RdbEtlService { columns.add(rsd.getColumnName(i)); } - Map columnsMap = SyncUtil.getColumnsMap(dbMapping, columns); + columnsMap.putAll(SyncUtil.getColumnsMap(dbMapping, columns)); + return true; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } + }); + + Util.sqlRS(srcDS, sql, rs -> { + int idx = 1; + + try { + boolean completed = false; + // if (dbMapping.isMapAll()) { // columnsMap = dbMapping.getAllColumns(); // } else { // columnsMap = dbMapping.getTargetColumns(); // } - StringBuilder insertSql = new StringBuilder(); - insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); - columnsMap.forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); + StringBuilder insertSql = new StringBuilder(); + insertSql.append("INSERT INTO ").append(SyncUtil.getDbTableName(dbMapping)).append(" ("); + columnsMap + .forEach((targetColumnName, srcColumnName) -> insertSql.append(targetColumnName).append(",")); - int len = insertSql.length(); - insertSql.delete(len - 1, len).append(") VALUES ("); - int mapLen = columnsMap.size(); - for (int i = 0; i < mapLen; i++) { - insertSql.append("?,"); - } - len = insertSql.length(); - insertSql.delete(len - 1, len).append(")"); - try (Connection connTarget = targetDS.getConnection(); - PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) { - connTarget.setAutoCommit(false); + int len = insertSql.length(); + insertSql.delete(len - 1, len).append(") VALUES ("); + int mapLen = columnsMap.size(); + for (int i = 0; i < mapLen; i++) { + insertSql.append("?,"); + } + len = insertSql.length(); + insertSql.delete(len - 1, len).append(")"); + try (Connection connTarget = targetDS.getConnection(); + PreparedStatement pstmt = connTarget.prepareStatement(insertSql.toString())) { + connTarget.setAutoCommit(false); - while (rs.next()) { - pstmt.clearParameters(); + while (rs.next()) { + pstmt.clearParameters(); - // 删除数据 - Map values = new LinkedHashMap<>(); - StringBuilder deleteSql = new StringBuilder("DELETE FROM " + SyncUtil.getDbTableName(dbMapping) - + " WHERE "); - appendCondition(dbMapping, deleteSql, values, rs); - try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) { - int k = 1; - for (Object val : values.values()) { - pstmt2.setObject(k++, val); - } - pstmt2.execute(); - } - - int i = 1; - for (Map.Entry entry : columnsMap.entrySet()) { - String targetClolumnName = entry.getKey(); - String srcColumnName = entry.getValue(); - if (srcColumnName == null) { - srcColumnName = targetClolumnName; + // 删除数据 + Map values = new LinkedHashMap<>(); + StringBuilder deleteSql = new StringBuilder( + "DELETE FROM " + SyncUtil.getDbTableName(dbMapping) + " WHERE "); + appendCondition(dbMapping, deleteSql, values, rs); + try (PreparedStatement pstmt2 = connTarget.prepareStatement(deleteSql.toString())) { + int k = 1; + for (Object val : values.values()) { + pstmt2.setObject(k++, val); + } + pstmt2.execute(); } - Integer type = columnType.get(targetClolumnName.toLowerCase()); + int i = 1; + for (Map.Entry entry : columnsMap.entrySet()) { + String targetClolumnName = entry.getKey(); + String srcColumnName = entry.getValue(); + if (srcColumnName == null) { + srcColumnName = targetClolumnName; + } - Object value = rs.getObject(srcColumnName); - if (value != null) { - SyncUtil.setPStmt(type, pstmt, value, i); - } else { - pstmt.setNull(i, type); + Integer type = columnType.get(targetClolumnName.toLowerCase()); + + Object value = rs.getObject(srcColumnName); + if (value != null) { + SyncUtil.setPStmt(type, pstmt, value, i); + } else { + pstmt.setNull(i, type); + } + + i++; } - i++; - } + pstmt.execute(); + if (logger.isTraceEnabled()) { + logger.trace("Insert into target table, sql: {}", insertSql); + } - pstmt.execute(); - if (logger.isTraceEnabled()) { - logger.trace("Insert into target table, sql: {}", insertSql); + if (idx % dbMapping.getCommitBatch() == 0) { + connTarget.commit(); + completed = true; + } + idx++; + successCount.incrementAndGet(); + if (logger.isDebugEnabled()) { + logger.debug("successful import count:" + successCount.get()); + } } - - if (idx % dbMapping.getCommitBatch() == 0) { + if (!completed) { connTarget.commit(); - completed = true; - } - idx++; - successCount.incrementAndGet(); - if (logger.isDebugEnabled()) { - logger.debug("successful import count:" + successCount.get()); } } - if (!completed) { - connTarget.commit(); - } - } - } catch (Exception e) { - logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e); - errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage()); - } - return idx; - } ); + } catch (Exception e) { + logger.error(dbMapping.getTable() + " etl failed! ==>" + e.getMessage(), e); + errMsg.add(dbMapping.getTable() + " etl failed! ==>" + e.getMessage()); + } + return idx; + }); return true; } catch (Exception e) { logger.error(e.getMessage(), e); @@ -278,8 +288,8 @@ public class RdbEtlService { /** * 拼接目标表主键where条件 */ - private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map values, ResultSet rs) - throws SQLException { + private static void appendCondition(DbMapping dbMapping, StringBuilder sql, Map values, + ResultSet rs) throws SQLException { // 拼接主键 for (Map.Entry entry : dbMapping.getTargetPk().entrySet()) { String targetColumnName = entry.getKey();