diff --git a/client-adapter/common/pom.xml b/client-adapter/common/pom.xml index 9c97646b..ec3ab9b8 100644 --- a/client-adapter/common/pom.xml +++ b/client-adapter/common/pom.xml @@ -1,5 +1,6 @@ - + canal.client-adapter com.alibaba.otter @@ -26,6 +27,16 @@ druid 1.1.9 + + org.springframework + spring-context + 5.0.5.RELEASE + + + org.yaml + snakeyaml + 1.19 + diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java index 37786222..31004f16 100644 --- a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/OuterAdapter.java @@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter; import java.util.List; import java.util.Map; +import java.util.Properties; import com.alibaba.otter.canal.client.adapter.support.Dml; import com.alibaba.otter.canal.client.adapter.support.EtlResult; @@ -21,8 +22,9 @@ public interface OuterAdapter { * 外部适配器初始化接口 * * @param configuration 外部适配器配置信息 + * @param envProperties 环境变量的配置属性 */ - void init(OuterAdapterConfig configuration); + void init(OuterAdapterConfig configuration, Properties envProperties); /** * 往适配器中同步数据 @@ -38,7 +40,7 @@ public interface OuterAdapter { /** * Etl操作 - * + * * @param task 任务名, 对应配置名 * @param params etl筛选条件 */ @@ -48,7 +50,7 @@ public interface OuterAdapter { /** * 计算总数 - * + * * @param task 任务名, 对应配置名 * @return 总数 */ @@ -58,7 +60,7 @@ public interface OuterAdapter { /** * 通过task获取对应的destination - * + * * @param task 任务名, 对应配置名 * @return destination */ diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java new file mode 100644 index 00000000..8d49e726 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java @@ -0,0 +1,148 @@ +package com.alibaba.otter.canal.client.adapter.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.springframework.util.PropertyPlaceholderHelper; +import org.springframework.util.StringUtils; + +import com.alibaba.otter.canal.client.adapter.config.bind.PropertiesConfigurationFactory; +import com.alibaba.otter.canal.client.adapter.config.common.*; + +/** + * 将yaml内容绑定到指定对象, 遵循spring yml的绑定规范 + * + * @author reweerma 2019-2-1 上午9:14:02 + * @version 1.0.0 + */ +public class YmlConfigBinder { + + /** + * 将当前内容绑定到指定对象 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz) { + return bindYmlToObj(null, content, clazz, null); + } + + /** + * 将当前内容绑定到指定对象并指定内容编码格式 + * + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String content, Class clazz, String charset) { + return bindYmlToObj(null, content, clazz, charset); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz) { + return bindYmlToObj(prefix, content, clazz, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并指定内容编码格式 + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset) { + return bindYmlToObj(prefix, content, clazz, charset, null); + } + + /** + * 将当前内容指定前缀部分绑定到指定对象并用环境变量中的属性替换占位符, 例: + * 当前内容有属性 zkServers: ${zookeeper.servers} + * 在envProperties中有属性 zookeeper.servers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 + * 则当前内容 zkServers 会被替换为 zkServers: 192.168.0.1:2181,192.168.0.1:2181,192.168.0.1:2181 + * 注: 假设绑定的类中 zkServers 属性是 List 对象, 则会自动映射成List + * + * @param prefix 指定前缀 + * @param content yml内容 + * @param clazz 指定对象类型 + * @param charset yml内容编码格式 + * @return 对象 + */ + public static T bindYmlToObj(String prefix, String content, Class clazz, String charset, + Properties baseProperties) { + try { + byte[] contentBytes; + if (charset == null) { + contentBytes = content.getBytes(); + } else { + contentBytes = content.getBytes(charset); + } + YamlPropertySourceLoader propertySourceLoader = new YamlPropertySourceLoader(); + Resource configResource = new ByteArrayResource(contentBytes); + PropertySource propertySource = propertySourceLoader.load("manualBindConfig", configResource, null); + + Properties properties = new Properties(); + Map propertiesRes = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(prefix) && !prefix.endsWith(".")) { + prefix = prefix + "."; + } + + properties.putAll((Map) propertySource.getSource()); + + if (baseProperties != null) { + baseProperties.putAll(properties); + properties = baseProperties; + } + + for (Object o : ((Map) propertySource.getSource()).entrySet()) { + Map.Entry entry = (Map.Entry) o; + String key = (String) entry.getKey(); + Object value = entry.getValue(); + + if (prefix != null) { + if (key != null && key.startsWith(prefix)) { + key = key.substring(prefix.length()); + } else { + continue; + } + } + + if (value != null && value.toString().contains("${")) { + PropertyPlaceholderHelper propertyPlaceholderHelper = new PropertyPlaceholderHelper("${", "}"); + value = propertyPlaceholderHelper.replacePlaceholders(value.toString(), properties); + } + + propertiesRes.put(key, value); + } + + propertySource = new MapPropertySource(propertySource.getName(), propertiesRes); + + T target = clazz.newInstance(); + + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(propertySource); + + PropertiesConfigurationFactory factory = new PropertiesConfigurationFactory(target); + factory.setPropertySources(propertySources); + factory.setIgnoreInvalidFields(true); + factory.setIgnoreUnknownFields(true); + + factory.bindPropertiesToTarget(); + + return target; + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..0b63ed2c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java @@ -0,0 +1,97 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final char[] delimiters; + + private final boolean ignoreCase; + + private final String[] names; + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, String... names){ + this(delimiters, false, names); + } + + protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){ + this(delimiters, ignoreCase, new HashSet(Arrays.asList(names))); + } + + DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set names){ + this.delimiters = delimiters; + this.ignoreCase = ignoreCase; + this.names = names.toArray(new String[names.size()]); + } + + @Override + public boolean matches(String propertyName) { + char[] propertyNameChars = propertyName.toCharArray(); + boolean[] match = new boolean[this.names.length]; + boolean noneMatched = true; + for (int i = 0; i < this.names.length; i++) { + if (this.names[i].length() <= propertyNameChars.length) { + match[i] = true; + noneMatched = false; + } + } + if (noneMatched) { + return false; + } + for (int charIndex = 0; charIndex < propertyNameChars.length; charIndex++) { + for (int nameIndex = 0; nameIndex < this.names.length; nameIndex++) { + if (match[nameIndex]) { + match[nameIndex] = false; + if (charIndex < this.names[nameIndex].length()) { + if (isCharMatch(this.names[nameIndex].charAt(charIndex), propertyNameChars[charIndex])) { + match[nameIndex] = true; + noneMatched = false; + } + } else { + char charAfter = propertyNameChars[this.names[nameIndex].length()]; + if (isDelimiter(charAfter)) { + match[nameIndex] = true; + noneMatched = false; + } + } + } + } + if (noneMatched) { + return false; + } + } + for (int i = 0; i < match.length; i++) { + if (match[i]) { + return true; + } + } + return false; + } + + private boolean isCharMatch(char c1, char c2) { + if (this.ignoreCase) { + return Character.toLowerCase(c1) == Character.toLowerCase(c2); + } + return c1 == c2; + } + + private boolean isDelimiter(char c) { + for (char delimiter : this.delimiters) { + if (c == delimiter) { + return true; + } + } + return false; + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java new file mode 100644 index 00000000..c9b259e3 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/InetAddressEditor.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditorSupport; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * {@link PropertyNamePatternsMatcher} that matches when a property name exactly + * matches one of the given names, or starts with one of the given names + * followed by a delimiter. This implementation is optimized for frequent calls. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class InetAddressEditor extends PropertyEditorSupport { + + @Override + public String getAsText() { + return ((InetAddress) getValue()).getHostAddress(); + } + + @Override + public void setAsText(String text) throws IllegalArgumentException { + try { + setValue(InetAddress.getByName(text)); + } catch (UnknownHostException ex) { + throw new IllegalArgumentException("Cannot locate host", ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java new file mode 100644 index 00000000..184d3d0c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/OriginCapablePropertyValue.java @@ -0,0 +1,52 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.beans.PropertyValue; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * A {@link PropertyValue} that can provide information about its origin. + * + * @author Andy Wilkinson + */ +class OriginCapablePropertyValue extends PropertyValue { + + private static final String ATTRIBUTE_PROPERTY_ORIGIN = "propertyOrigin"; + + private final PropertyOrigin origin; + + OriginCapablePropertyValue(PropertyValue propertyValue){ + this(propertyValue.getName(), + propertyValue.getValue(), + (PropertyOrigin) propertyValue.getAttribute(ATTRIBUTE_PROPERTY_ORIGIN)); + } + + OriginCapablePropertyValue(String name, Object value, String originName, PropertySource originSource){ + this(name, value, new PropertyOrigin(originSource, originName)); + } + + OriginCapablePropertyValue(String name, Object value, PropertyOrigin origin){ + super(name, value); + this.origin = origin; + setAttribute(ATTRIBUTE_PROPERTY_ORIGIN, origin); + } + + public PropertyOrigin getOrigin() { + return this.origin; + } + + @Override + public String toString() { + String name = (this.origin != null ? this.origin.getName() : this.getName()); + String source = (this.origin.getSource() != null ? this.origin.getSource().getName() : "unknown"); + return "'" + name + "' from '" + source + "'"; + } + + public static PropertyOrigin getOrigin(PropertyValue propertyValue) { + if (propertyValue instanceof OriginCapablePropertyValue) { + return ((OriginCapablePropertyValue) propertyValue).getOrigin(); + } + return new OriginCapablePropertyValue(propertyValue).getOrigin(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java new file mode 100644 index 00000000..a041a673 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PatternPropertyNamePatternsMatcher.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; + +import org.springframework.util.PatternMatchUtils; + +/** + * {@link PropertyNamePatternsMatcher} that delegates to + * {@link PatternMatchUtils#simpleMatch(String[], String)}. + * + * @author Phillip Webb + * @since 1.2.0 + */ +class PatternPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher { + + private final String[] patterns; + + PatternPropertyNamePatternsMatcher(Collection patterns){ + this.patterns = (patterns != null ? patterns.toArray(new String[patterns.size()]) : new String[] {}); + } + + @Override + public boolean matches(String propertyName) { + return PatternMatchUtils.simpleMatch(this.patterns, propertyName); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java new file mode 100644 index 00000000..76a97c0b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java @@ -0,0 +1,356 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyDescriptor; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.PropertyValues; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.support.ResourceEditorRegistrar; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.MessageSource; +import org.springframework.context.MessageSourceAware; +import org.springframework.core.convert.ConversionService; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.validation.*; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * Validate some {@link Properties} (or optionally + * {@link org.springframework.core.env.PropertySources}) by binding them to an + * object of a specified type and then optionally running a {@link Validator} + * over it. + * + * @param the target type + * @author Dave Syer + */ +public class PropertiesConfigurationFactory implements FactoryBean, ApplicationContextAware, MessageSourceAware, InitializingBean { + + private static final char[] EXACT_DELIMITERS = { '_', '.', '[' }; + + private static final char[] TARGET_NAME_DELIMITERS = { '_', '.' }; + + private static final Log logger = LogFactory.getLog(PropertiesConfigurationFactory.class); + + private boolean ignoreUnknownFields = true; + + private boolean ignoreInvalidFields; + + private boolean exceptionIfInvalid = true; + + private PropertySources propertySources; + + private final T target; + + private Validator validator; + + private ApplicationContext applicationContext; + + private MessageSource messageSource; + + private boolean hasBeenBound = false; + + private boolean ignoreNestedProperties = false; + + private String targetName; + + private ConversionService conversionService; + + private boolean resolvePlaceholders = true; + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param target the target object to bind too + * @see #PropertiesConfigurationFactory(Class) + */ + public PropertiesConfigurationFactory(T target){ + Assert.notNull(target, "target must not be null"); + this.target = target; + } + + /** + * Create a new {@link PropertiesConfigurationFactory} instance. + * + * @param type the target type + * @see #PropertiesConfigurationFactory(Class) + */ + @SuppressWarnings("unchecked") + public PropertiesConfigurationFactory(Class type){ + Assert.notNull(type, "type must not be null"); + this.target = (T) BeanUtils.instantiate(type); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set whether to ignore unknown fields, that is, whether to ignore bind + * parameters that do not have corresponding fields in the target object. + *

+ * Default is "true". Turn this off to enforce that all bind parameters must + * have a matching field in the target object. + * + * @param ignoreUnknownFields if unknown fields should be ignored + */ + public void setIgnoreUnknownFields(boolean ignoreUnknownFields) { + this.ignoreUnknownFields = ignoreUnknownFields; + } + + /** + * Set whether to ignore invalid fields, that is, whether to ignore bind + * parameters that have corresponding fields in the target object which are not + * accessible (for example because of null values in the nested path). + *

+ * Default is "false". Turn this on to ignore bind parameters for nested objects + * in non-existing parts of the target object graph. + * + * @param ignoreInvalidFields if invalid fields should be ignored + */ + public void setIgnoreInvalidFields(boolean ignoreInvalidFields) { + this.ignoreInvalidFields = ignoreInvalidFields; + } + + /** + * Set the target name. + * + * @param targetName the target name + */ + public void setTargetName(String targetName) { + this.targetName = targetName; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + } + + /** + * Set the message source. + * + * @param messageSource the message source + */ + @Override + public void setMessageSource(MessageSource messageSource) { + this.messageSource = messageSource; + } + + /** + * Set the property sources. + * + * @param propertySources the property sources + */ + public void setPropertySources(PropertySources propertySources) { + this.propertySources = propertySources; + } + + /** + * Set the conversion service. + * + * @param conversionService the conversion service + */ + public void setConversionService(ConversionService conversionService) { + this.conversionService = conversionService; + } + + /** + * Set the validator. + * + * @param validator the validator + */ + public void setValidator(Validator validator) { + this.validator = validator; + } + + /** + * Set a flag to indicate that an exception should be raised if a Validator is + * available and validation fails. + * + * @param exceptionIfInvalid the flag to set + * @deprecated as of 1.5, do not specify a {@link Validator} if validation + * should not occur + */ + @Deprecated + public void setExceptionIfInvalid(boolean exceptionIfInvalid) { + this.exceptionIfInvalid = exceptionIfInvalid; + } + + /** + * Flag to indicate that placeholders should be replaced during binding. Default + * is true. + * + * @param resolvePlaceholders flag value + */ + public void setResolvePlaceholders(boolean resolvePlaceholders) { + this.resolvePlaceholders = resolvePlaceholders; + } + + @Override + public void afterPropertiesSet() throws Exception { + bindPropertiesToTarget(); + } + + @Override + public Class getObjectType() { + if (this.target == null) { + return Object.class; + } + return this.target.getClass(); + } + + @Override + public boolean isSingleton() { + return true; + } + + @Override + public T getObject() throws Exception { + if (!this.hasBeenBound) { + bindPropertiesToTarget(); + } + return this.target; + } + + public void bindPropertiesToTarget() throws BindException { + Assert.state(this.propertySources != null, "PropertySources should not be null"); + try { + if (logger.isTraceEnabled()) { + logger.trace("Property Sources: " + this.propertySources); + + } + this.hasBeenBound = true; + doBindPropertiesToTarget(); + } catch (BindException ex) { + if (this.exceptionIfInvalid) { + throw ex; + } + logger.error("Failed to load Properties validation bean. " + "Your Properties may be invalid.", ex); + } + } + + private void doBindPropertiesToTarget() throws BindException { + RelaxedDataBinder dataBinder = (this.targetName != null ? new RelaxedDataBinder(this.target, + this.targetName) : new RelaxedDataBinder(this.target)); + if (this.validator != null && this.validator.supports(dataBinder.getTarget().getClass())) { + dataBinder.setValidator(this.validator); + } + if (this.conversionService != null) { + dataBinder.setConversionService(this.conversionService); + } + dataBinder.setAutoGrowCollectionLimit(Integer.MAX_VALUE); + dataBinder.setIgnoreNestedProperties(this.ignoreNestedProperties); + dataBinder.setIgnoreInvalidFields(this.ignoreInvalidFields); + dataBinder.setIgnoreUnknownFields(this.ignoreUnknownFields); + customizeBinder(dataBinder); + if (this.applicationContext != null) { + ResourceEditorRegistrar resourceEditorRegistrar = new ResourceEditorRegistrar(this.applicationContext, + this.applicationContext.getEnvironment()); + resourceEditorRegistrar.registerCustomEditors(dataBinder); + } + Iterable relaxedTargetNames = getRelaxedTargetNames(); + Set names = getNames(relaxedTargetNames); + PropertyValues propertyValues = getPropertySourcesPropertyValues(names, relaxedTargetNames); + dataBinder.bind(propertyValues); + if (this.validator != null) { + dataBinder.validate(); + } + checkForBindingErrors(dataBinder); + } + + private Iterable getRelaxedTargetNames() { + return (this.target != null + && StringUtils.hasLength(this.targetName) ? new RelaxedNames(this.targetName) : null); + } + + private Set getNames(Iterable prefixes) { + Set names = new LinkedHashSet(); + if (this.target != null) { + PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass()); + for (PropertyDescriptor descriptor : descriptors) { + String name = descriptor.getName(); + if (!name.equals("class")) { + RelaxedNames relaxedNames = RelaxedNames.forCamelCase(name); + if (prefixes == null) { + for (String relaxedName : relaxedNames) { + names.add(relaxedName); + } + } else { + for (String prefix : prefixes) { + for (String relaxedName : relaxedNames) { + names.add(prefix + "." + relaxedName); + names.add(prefix + "_" + relaxedName); + } + } + } + } + } + } + return names; + } + + private PropertyValues getPropertySourcesPropertyValues(Set names, Iterable relaxedTargetNames) { + PropertyNamePatternsMatcher includes = getPropertyNamePatternsMatcher(names, relaxedTargetNames); + return new PropertySourcesPropertyValues(this.propertySources, names, includes, this.resolvePlaceholders); + } + + private PropertyNamePatternsMatcher getPropertyNamePatternsMatcher(Set names, + Iterable relaxedTargetNames) { + if (this.ignoreUnknownFields && !isMapTarget()) { + // Since unknown fields are ignored we can filter them out early to save + // unnecessary calls to the PropertySource. + return new DefaultPropertyNamePatternsMatcher(EXACT_DELIMITERS, true, names); + } + if (relaxedTargetNames != null) { + // We can filter properties to those starting with the target name, but + // we can't do a complete filter since we need to trigger the + // unknown fields check + Set relaxedNames = new HashSet(); + for (String relaxedTargetName : relaxedTargetNames) { + relaxedNames.add(relaxedTargetName); + } + return new DefaultPropertyNamePatternsMatcher(TARGET_NAME_DELIMITERS, true, relaxedNames); + } + // Not ideal, we basically can't filter anything + return PropertyNamePatternsMatcher.ALL; + } + + private boolean isMapTarget() { + return this.target != null && Map.class.isAssignableFrom(this.target.getClass()); + } + + private void checkForBindingErrors(RelaxedDataBinder dataBinder) throws BindException { + BindingResult errors = dataBinder.getBindingResult(); + if (errors.hasErrors()) { + logger.error("Properties configuration failed validation"); + for (ObjectError error : errors.getAllErrors()) { + logger.error(this.messageSource != null ? this.messageSource.getMessage(error, Locale.getDefault()) + + " (" + error + ")" : error); + } + if (this.exceptionIfInvalid) { + throw new BindException(errors); + } + } + } + + /** + * Customize the data binder. + * + * @param dataBinder the data binder that will be used to bind and validate + */ + protected void customizeBinder(DataBinder dataBinder) { + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java new file mode 100644 index 00000000..0d7d4fe4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +/** + * Strategy interface used to check if a property name matches specific + * criteria. + * + * @author Phillip Webb + * @since 1.2.0 + */ +interface PropertyNamePatternsMatcher { + + PropertyNamePatternsMatcher ALL = new PropertyNamePatternsMatcher() { + + @Override + public boolean matches(String propertyName) { + return true; + } + + }; + + PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() { + + @Override + public boolean matches(String propertyName) { + return false; + } + + }; + + /** + * Return {@code true} of the property name matches. + * + * @param propertyName the property name + * @return {@code true} if the property name matches + */ + boolean matches(String propertyName); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java new file mode 100644 index 00000000..7ad35f1a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyOrigin.java @@ -0,0 +1,30 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; + +/** + * The origin of a property, specifically its source and its name before any + * prefix was removed. + * + * @author Andy Wilkinson + * @since 1.3.0 + */ +public class PropertyOrigin { + + private final PropertySource source; + + private final String name; + + PropertyOrigin(PropertySource source, String name){ + this.name = name; + this.source = source; + } + + public PropertySource getSource() { + return this.source; + } + + public String getName() { + return this.name; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java new file mode 100644 index 00000000..520e857f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyResolver.java @@ -0,0 +1,164 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.ConversionException; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.core.env.AbstractPropertyResolver; +import org.springframework.core.env.PropertyResolver; +import org.springframework.util.ClassUtils; + +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * {@link PropertyResolver} implementation that resolves property values against + * an underlying set of {@link PropertySources}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySource + * @see PropertySources + * @see AbstractEnvironment + * @since 3.1 + */ +public class PropertySourcesPropertyResolver extends AbstractPropertyResolver { + + private final PropertySources propertySources; + + /** + * Create a new resolver against the given property sources. + * + * @param propertySources the set of {@link PropertySource} objects to use + */ + public PropertySourcesPropertyResolver(PropertySources propertySources){ + this.propertySources = propertySources; + } + + @Override + public boolean containsProperty(String key) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(key)) { + return true; + } + } + } + return false; + } + + @Override + public String getProperty(String key) { + return getProperty(key, String.class, true); + } + + @Override + public T getProperty(String key, Class targetValueType) { + return getProperty(key, targetValueType, true); + } + + @Override + protected String getPropertyAsRawString(String key) { + return getProperty(key, String.class, false); + } + + protected T getProperty(String key, Class targetValueType, boolean resolveNestedPlaceholders) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger + .trace("Searching for key '" + key + "' in PropertySource '" + propertySource.getName() + "'"); + } + Object value = propertySource.getProperty(key); + if (value != null) { + if (resolveNestedPlaceholders && value instanceof String) { + value = resolveNestedPlaceholders((String) value); + } + logKeyFound(key, propertySource, value); + return convertValueIfNecessary(value, targetValueType); + } + } + } + if (logger.isDebugEnabled()) { + logger.debug("Could not find key '" + key + "' in any property source"); + } + return null; + } + + @Deprecated + public Class getPropertyAsClass(String key, Class targetValueType) { + if (this.propertySources != null) { + for (PropertySource propertySource : this.propertySources) { + if (logger.isTraceEnabled()) { + logger.trace(String.format("Searching for key '%s' in [%s]", key, propertySource.getName())); + } + Object value = propertySource.getProperty(key); + if (value != null) { + logKeyFound(key, propertySource, value); + Class clazz; + if (value instanceof String) { + try { + clazz = ClassUtils.forName((String) value, null); + } catch (Exception ex) { + throw new PropertySourcesPropertyResolver.ClassConversionException((String) value, + targetValueType, + ex); + } + } else if (value instanceof Class) { + clazz = (Class) value; + } else { + clazz = value.getClass(); + } + if (!targetValueType.isAssignableFrom(clazz)) { + throw new PropertySourcesPropertyResolver.ClassConversionException(clazz, targetValueType); + } + @SuppressWarnings("unchecked") + Class targetClass = (Class) clazz; + return targetClass; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug(String.format("Could not find key '%s' in any property source", key)); + } + return null; + } + + /** + * Log the given key as found in the given {@link PropertySource}, resulting in + * the given value. + *

+ * The default implementation writes a debug log message with key and source. As + * of 4.3.3, this does not log the value anymore in order to avoid accidental + * logging of sensitive settings. Subclasses may override this method to change + * the log level and/or log message, including the property's value if desired. + * + * @param key the key found + * @param propertySource the {@code PropertySource} that the key has been found + * in + * @param value the corresponding value + * @since 4.3.1 + */ + protected void logKeyFound(String key, PropertySource propertySource, Object value) { + if (logger.isDebugEnabled()) { + logger.debug("Found key '" + key + "' in PropertySource '" + propertySource.getName() + + "' with value of type " + value.getClass().getSimpleName()); + } + } + + @SuppressWarnings("serial") + @Deprecated + private static class ClassConversionException extends ConversionException { + + public ClassConversionException(Class actual, Class expected){ + super(String + .format("Actual type %s is not assignable to expected type %s", actual.getName(), expected.getName())); + } + + public ClassConversionException(String actual, Class expected, Exception ex){ + super( + String + .format("Could not find/load class %s during attempt to convert to %s", actual, expected.getName()), + ex); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java new file mode 100644 index 00000000..d41cb27b --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java @@ -0,0 +1,233 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.util.Assert; +import org.springframework.validation.DataBinder; + +import com.alibaba.otter.canal.client.adapter.config.common.CompositePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.EnumerablePropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySource; +import com.alibaba.otter.canal.client.adapter.config.common.PropertySources; + +/** + * A {@link PropertyValues} implementation backed by a {@link PropertySources}, + * bridging the two abstractions and allowing (for instance) a regular + * {@link DataBinder} to be used with the latter. + * + * @author Dave Syer + * @author Phillip Webb + */ +public class PropertySourcesPropertyValues implements PropertyValues { + + private static final Pattern COLLECTION_PROPERTY = Pattern + .compile("\\[(\\d+)\\](\\.\\S+)?"); + + private final PropertySources propertySources; + + private final Collection nonEnumerableFallbackNames; + + private final PropertyNamePatternsMatcher includes; + + private final Map propertyValues = new LinkedHashMap(); + + private final ConcurrentHashMap> collectionOwners = new ConcurrentHashMap>(); + + private final boolean resolvePlaceholders; + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + */ + public PropertySourcesPropertyValues(PropertySources propertySources){ + this(propertySources, true); + } + + /** + * Create a new PropertyValues from the given PropertySources that will + * optionally resolve placeholders. + * + * @param propertySources a PropertySources instance + * @param resolvePlaceholders {@code true} if placeholders should be resolved. + * @since 1.5.2 + */ + public PropertySourcesPropertyValues(PropertySources propertySources, boolean resolvePlaceholders){ + this(propertySources, (Collection) null, PropertyNamePatternsMatcher.ALL, resolvePlaceholders); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param includePatterns property name patterns to include from system + * properties and environment variables + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + */ + public PropertySourcesPropertyValues(PropertySources propertySources, Collection includePatterns, + Collection nonEnumerableFallbackNames){ + this(propertySources, + nonEnumerableFallbackNames, + new PatternPropertyNamePatternsMatcher(includePatterns), + true); + } + + /** + * Create a new PropertyValues from the given PropertySources. + * + * @param propertySources a PropertySources instance + * @param nonEnumerableFallbackNames the property names to try in lieu of an + * {@link EnumerablePropertySource}. + * @param includes the property name patterns to include + * @param resolvePlaceholders flag to indicate the placeholders should be + * resolved + */ + PropertySourcesPropertyValues(PropertySources propertySources, Collection nonEnumerableFallbackNames, + PropertyNamePatternsMatcher includes, boolean resolvePlaceholders){ + Assert.notNull(propertySources, "PropertySources must not be null"); + Assert.notNull(includes, "Includes must not be null"); + this.propertySources = propertySources; + this.nonEnumerableFallbackNames = nonEnumerableFallbackNames; + this.includes = includes; + this.resolvePlaceholders = resolvePlaceholders; + PropertySourcesPropertyResolver resolver = new PropertySourcesPropertyResolver(propertySources); + for (PropertySource source : propertySources) { + processPropertySource(source, resolver); + } + } + + private void processPropertySource(PropertySource source, PropertySourcesPropertyResolver resolver) { + if (source instanceof CompositePropertySource) { + processCompositePropertySource((CompositePropertySource) source, resolver); + } else if (source instanceof EnumerablePropertySource) { + processEnumerablePropertySource((EnumerablePropertySource) source, resolver, this.includes); + } else { + processNonEnumerablePropertySource(source, resolver); + } + } + + private void processCompositePropertySource(CompositePropertySource source, + PropertySourcesPropertyResolver resolver) { + for (PropertySource nested : source.getPropertySources()) { + processPropertySource(nested, resolver); + } + } + + private void processEnumerablePropertySource(EnumerablePropertySource source, + PropertySourcesPropertyResolver resolver, + PropertyNamePatternsMatcher includes) { + if (source.getPropertyNames().length > 0) { + for (String propertyName : source.getPropertyNames()) { + if (includes.matches(propertyName)) { + Object value = getEnumerableProperty(source, resolver, propertyName); + putIfAbsent(propertyName, value, source); + } + } + } + } + + private Object getEnumerableProperty(EnumerablePropertySource source, PropertySourcesPropertyResolver resolver, + String propertyName) { + try { + if (this.resolvePlaceholders) { + return resolver.getProperty(propertyName, Object.class); + } + } catch (RuntimeException ex) { + // Probably could not resolve placeholders, ignore it here + } + return source.getProperty(propertyName); + } + + private void processNonEnumerablePropertySource(PropertySource source, + PropertySourcesPropertyResolver resolver) { + // We can only do exact matches for non-enumerable property names, but + // that's better than nothing... + if (this.nonEnumerableFallbackNames == null) { + return; + } + for (String propertyName : this.nonEnumerableFallbackNames) { + if (!source.containsProperty(propertyName)) { + continue; + } + Object value = null; + try { + value = resolver.getProperty(propertyName, Object.class); + } catch (RuntimeException ex) { + // Probably could not convert to Object, weird, but ignorable + } + if (value == null) { + value = source.getProperty(propertyName.toUpperCase(Locale.ENGLISH)); + } + putIfAbsent(propertyName, value, source); + } + } + + @Override + public PropertyValue[] getPropertyValues() { + Collection values = this.propertyValues.values(); + return values.toArray(new PropertyValue[values.size()]); + } + + @Override + public PropertyValue getPropertyValue(String propertyName) { + PropertyValue propertyValue = this.propertyValues.get(propertyName); + if (propertyValue != null) { + return propertyValue; + } + for (PropertySource source : this.propertySources) { + Object value = source.getProperty(propertyName); + propertyValue = putIfAbsent(propertyName, value, source); + if (propertyValue != null) { + return propertyValue; + } + } + return null; + } + + private PropertyValue putIfAbsent(String propertyName, Object value, PropertySource source) { + if (value != null && !this.propertyValues.containsKey(propertyName)) { + PropertySource collectionOwner = this.collectionOwners + .putIfAbsent(COLLECTION_PROPERTY.matcher(propertyName).replaceAll("[]"), source); + if (collectionOwner == null || collectionOwner == source) { + PropertyValue propertyValue = new OriginCapablePropertyValue(propertyName, value, propertyName, source); + this.propertyValues.put(propertyName, propertyValue); + return propertyValue; + } + } + return null; + } + + @Override + public PropertyValues changesSince(PropertyValues old) { + MutablePropertyValues changes = new MutablePropertyValues(); + // for each property value in the new set + for (PropertyValue newValue : getPropertyValues()) { + // if there wasn't an old one, add it + PropertyValue oldValue = old.getPropertyValue(newValue.getName()); + if (oldValue == null || !oldValue.equals(newValue)) { + changes.addPropertyValue(newValue); + } + } + return changes; + } + + @Override + public boolean contains(String propertyName) { + return getPropertyValue(propertyName) != null; + } + + @Override + public boolean isEmpty() { + return this.propertyValues.isEmpty(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java new file mode 100644 index 00000000..00a288b6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedConversionService.java @@ -0,0 +1,127 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; + +import org.springframework.core.convert.ConversionFailedException; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.convert.converter.ConverterFactory; +import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.util.Assert; + +/** + * Internal {@link ConversionService} used by {@link RelaxedDataBinder} to + * support additional relaxed conversion. + * + * @author Phillip Webb + * @author Stephane Nicoll + * @since 1.1.0 + */ +class RelaxedConversionService implements ConversionService { + + private final ConversionService conversionService; + + private final GenericConversionService additionalConverters; + + /** + * Create a new {@link RelaxedConversionService} instance. + * + * @param conversionService and option root conversion service + */ + RelaxedConversionService(ConversionService conversionService){ + this.conversionService = conversionService; + this.additionalConverters = new GenericConversionService(); + DefaultConversionService.addCollectionConverters(this.additionalConverters); + this.additionalConverters + .addConverterFactory(new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory()); + this.additionalConverters.addConverter(new StringToCharArrayConverter()); + } + + @Override + public boolean canConvert(Class sourceType, Class targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) { + return (this.conversionService != null && this.conversionService.canConvert(sourceType, targetType)) + || this.additionalConverters.canConvert(sourceType, targetType); + } + + @Override + @SuppressWarnings("unchecked") + public T convert(Object source, Class targetType) { + Assert.notNull(targetType, "The targetType to convert to cannot be null"); + return (T) convert(source, TypeDescriptor.forObject(source), TypeDescriptor.valueOf(targetType)); + } + + @Override + public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { + if (this.conversionService != null) { + try { + return this.conversionService.convert(source, sourceType, targetType); + } catch (ConversionFailedException ex) { + // Ignore and try the additional converters + } + } + return this.additionalConverters.convert(source, sourceType, targetType); + } + + /** + * Clone of Spring's package private StringToEnumConverterFactory, but ignoring + * the case of the source. + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private static class StringToEnumIgnoringCaseConverterFactory implements ConverterFactory { + + @Override + public Converter getConverter(Class targetType) { + Class enumType = targetType; + while (enumType != null && !enumType.isEnum()) { + enumType = enumType.getSuperclass(); + } + Assert.notNull(enumType, "The target type " + targetType.getName() + " does not refer to an enum"); + return new RelaxedConversionService.StringToEnumIgnoringCaseConverterFactory.StringToEnum(enumType); + } + + private class StringToEnum implements Converter { + + private final Class enumType; + + StringToEnum(Class enumType){ + this.enumType = enumType; + } + + @Override + public T convert(String source) { + if (source.isEmpty()) { + // It's an empty enum identifier: reset the enum value to null. + return null; + } + source = source.trim(); + for (T candidate : (Set) EnumSet.allOf(this.enumType)) { + RelaxedNames names = new RelaxedNames( + candidate.name().replace('_', '-').toLowerCase(Locale.ENGLISH)); + for (String name : names) { + if (name.equals(source)) { + return candidate; + } + } + if (candidate.name().equalsIgnoreCase(source)) { + return candidate; + } + } + throw new IllegalArgumentException( + "No enum constant " + this.enumType.getCanonicalName() + "." + source); + } + + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java new file mode 100644 index 00000000..dbcc7130 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java @@ -0,0 +1,729 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.beans.PropertyEditor; +import java.net.InetAddress; +import java.util.*; + +import org.springframework.beans.*; +import org.springframework.beans.propertyeditors.FileEditor; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; +import org.springframework.validation.AbstractPropertyBindingResult; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.DataBinder; + +/** + * Binder implementation that allows caller to bind to maps and also allows + * property names to match a bit loosely (if underscores or dashes are removed + * and replaced with camel case for example). + * + * @author Dave Syer + * @author Phillip Webb + * @author Stephane Nicoll + * @author Andy Wilkinson + * @see RelaxedNames + */ +public class RelaxedDataBinder extends DataBinder { + + private static final Set> EXCLUDED_EDITORS; + + static { + Set> excluded = new HashSet>(); + excluded.add(FileEditor.class); + EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded); + } + + private static final Object BLANK = new Object(); + + private String namePrefix; + + private boolean ignoreNestedProperties; + + private MultiValueMap nameAliases = new LinkedMultiValueMap(); + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + */ + public RelaxedDataBinder(Object target){ + super(wrapTarget(target)); + } + + /** + * Create a new {@link RelaxedDataBinder} instance. + * + * @param target the target into which properties are bound + * @param namePrefix An optional prefix to be used when reading properties + */ + public RelaxedDataBinder(Object target, String namePrefix){ + super(wrapTarget(target), (StringUtils.hasLength(namePrefix) ? namePrefix : DEFAULT_OBJECT_NAME)); + this.namePrefix = cleanNamePrefix(namePrefix); + } + + private String cleanNamePrefix(String namePrefix) { + if (!StringUtils.hasLength(namePrefix)) { + return null; + } + return (namePrefix.endsWith(".") ? namePrefix : namePrefix + "."); + } + + /** + * Flag to disable binding of nested properties (i.e. those with period + * separators in their paths). Can be useful to disable this if the name prefix + * is empty and you don't want to ignore unknown fields. + * + * @param ignoreNestedProperties the flag to set (default false) + */ + public void setIgnoreNestedProperties(boolean ignoreNestedProperties) { + this.ignoreNestedProperties = ignoreNestedProperties; + } + + /** + * Set name aliases. + * + * @param aliases a map of property name to aliases + */ + public void setNameAliases(Map> aliases) { + this.nameAliases = new LinkedMultiValueMap(aliases); + } + + /** + * Add aliases to the {@link DataBinder}. + * + * @param name the property name to alias + * @param alias aliases for the property names + * @return this instance + */ + public RelaxedDataBinder withAlias(String name, String... alias) { + for (String value : alias) { + this.nameAliases.add(name, value); + } + return this; + } + + @Override + protected void doBind(MutablePropertyValues propertyValues) { + super.doBind(modifyProperties(propertyValues, getTarget())); + } + + /** + * Modify the property values so that period separated property paths are valid + * for map keys. Also creates new maps for properties of map type that are null + * (assuming all maps are potentially nested). The standard bracket {@code[...]} + * dereferencing is also accepted. + * + * @param propertyValues the property values + * @param target the target object + * @return modified property values + */ + private MutablePropertyValues modifyProperties(MutablePropertyValues propertyValues, Object target) { + propertyValues = getPropertyValuesForNamePrefix(propertyValues); + if (target instanceof RelaxedDataBinder.MapHolder) { + propertyValues = addMapPrefix(propertyValues); + } + BeanWrapper wrapper = new BeanWrapperImpl(target); + wrapper.setConversionService(new RelaxedConversionService(getConversionService())); + wrapper.setAutoGrowNestedPaths(true); + List sortedValues = new ArrayList(); + Set modifiedNames = new HashSet(); + List sortedNames = getSortedPropertyNames(propertyValues); + for (String name : sortedNames) { + PropertyValue propertyValue = propertyValues.getPropertyValue(name); + PropertyValue modifiedProperty = modifyProperty(wrapper, propertyValue); + if (modifiedNames.add(modifiedProperty.getName())) { + sortedValues.add(modifiedProperty); + } + } + return new MutablePropertyValues(sortedValues); + } + + private List getSortedPropertyNames(MutablePropertyValues propertyValues) { + List names = new LinkedList(); + for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) { + names.add(propertyValue.getName()); + } + sortPropertyNames(names); + return names; + } + + /** + * Sort by name so that parent properties get processed first (e.g. 'foo.bar' + * before 'foo.bar.spam'). Don't use Collections.sort() because the order might + * be significant for other property names (it shouldn't be but who knows what + * people might be relying on, e.g. HSQL has a JDBCXADataSource where + * "databaseName" is a synonym for "url"). + * + * @param names the names to sort + */ + private void sortPropertyNames(List names) { + for (String name : new ArrayList(names)) { + int propertyIndex = names.indexOf(name); + RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name); + for (String prefix : path.prefixes()) { + int prefixIndex = names.indexOf(prefix); + if (prefixIndex >= propertyIndex) { + // The child property has a parent in the list in the wrong order + names.remove(name); + names.add(prefixIndex, name); + } + } + } + } + + private MutablePropertyValues addMapPrefix(MutablePropertyValues propertyValues) { + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue pv : propertyValues.getPropertyValues()) { + rtn.add("map." + pv.getName(), pv.getValue()); + } + return rtn; + } + + private MutablePropertyValues getPropertyValuesForNamePrefix(MutablePropertyValues propertyValues) { + if (!StringUtils.hasText(this.namePrefix) && !this.ignoreNestedProperties) { + return propertyValues; + } + MutablePropertyValues rtn = new MutablePropertyValues(); + for (PropertyValue value : propertyValues.getPropertyValues()) { + String name = value.getName(); + for (String prefix : new RelaxedNames(stripLastDot(this.namePrefix))) { + for (String separator : new String[] { ".", "_" }) { + String candidate = (StringUtils.hasLength(prefix) ? prefix + separator : prefix); + if (name.startsWith(candidate)) { + name = name.substring(candidate.length()); + if (!(this.ignoreNestedProperties && name.contains("."))) { + PropertyOrigin propertyOrigin = OriginCapablePropertyValue.getOrigin(value); + rtn.addPropertyValue( + new OriginCapablePropertyValue(name, value.getValue(), propertyOrigin)); + } + } + } + } + } + return rtn; + } + + private String stripLastDot(String string) { + if (StringUtils.hasLength(string) && string.endsWith(".")) { + string = string.substring(0, string.length() - 1); + } + return string; + } + + private PropertyValue modifyProperty(BeanWrapper target, PropertyValue propertyValue) { + String name = propertyValue.getName(); + String normalizedName = normalizePath(target, name); + if (!normalizedName.equals(name)) { + return new PropertyValue(normalizedName, propertyValue.getValue()); + } + return propertyValue; + } + + /** + * Normalize a bean property path to a format understood by a BeanWrapper. This + * is used so that + *

    + *
  • Fuzzy matching can be employed for bean property names
  • + *
  • Period separators can be used instead of indexing ([...]) for map + * keys
  • + *
+ * + * @param wrapper a bean wrapper for the object to bind + * @param path the bean path to bind + * @return a transformed path with correct bean wrapper syntax + */ + protected String normalizePath(BeanWrapper wrapper, String path) { + return initializePath(wrapper, new RelaxedDataBinder.BeanPath(path), 0); + } + + @Override + protected AbstractPropertyBindingResult createBeanPropertyBindingResult() { + return new RelaxedDataBinder.RelaxedBeanPropertyBindingResult(getTarget(), + getObjectName(), + isAutoGrowNestedPaths(), + getAutoGrowCollectionLimit(), + getConversionService()); + } + + private String initializePath(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String prefix = path.prefix(index); + String key = path.name(index); + if (path.isProperty(index)) { + key = getActualPropertyName(wrapper, prefix, key); + path.rename(index, key); + } + if (path.name(++index) == null) { + return path.toString(); + } + String name = path.prefix(index); + TypeDescriptor descriptor = wrapper.getPropertyTypeDescriptor(name); + if (descriptor == null || descriptor.isMap()) { + if (isMapValueStringType(descriptor) || isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + extendMapIfNecessary(wrapper, path, index); + } else if (descriptor.isCollection()) { + extendCollectionIfNecessary(wrapper, path, index); + } else if (descriptor.getType().equals(Object.class)) { + if (isBlanked(wrapper, name, path.name(index))) { + path.collapseKeys(index); + } + path.mapIndex(index); + if (path.isLastNode(index)) { + wrapper.setPropertyValue(path.toString(), BLANK); + } else { + String next = path.prefix(index + 1); + if (wrapper.getPropertyValue(next) == null) { + wrapper.setPropertyValue(next, new LinkedHashMap()); + } + } + } + return initializePath(wrapper, path, index); + } + + private boolean isMapValueStringType(TypeDescriptor descriptor) { + if (descriptor == null || descriptor.getMapValueTypeDescriptor() == null) { + return false; + } + if (Properties.class.isAssignableFrom(descriptor.getObjectType())) { + // Properties is declared as Map but we know it's really + // Map + return true; + } + Class valueType = descriptor.getMapValueTypeDescriptor().getObjectType(); + return (valueType != null && CharSequence.class.isAssignableFrom(valueType)); + } + + @SuppressWarnings("rawtypes") + private boolean isBlanked(BeanWrapper wrapper, String propertyName, String key) { + Object value = (wrapper.isReadableProperty(propertyName) ? wrapper.getPropertyValue(propertyName) : null); + if (value instanceof Map) { + if (((Map) value).get(key) == BLANK) { + return true; + } + } + return false; + } + + private void extendCollectionIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor elementDescriptor = wrapper.getPropertyTypeDescriptor(name).getElementTypeDescriptor(); + if (!elementDescriptor.isMap() && !elementDescriptor.isCollection() + && !elementDescriptor.getType().equals(Object.class)) { + return; + } + Object extend = new LinkedHashMap(); + if (!elementDescriptor.isMap() && path.isArrayIndex(index)) { + extend = new ArrayList(); + } + wrapper.setPropertyValue(path.prefix(index + 1), extend); + } + + private void extendMapIfNecessary(BeanWrapper wrapper, RelaxedDataBinder.BeanPath path, int index) { + String name = path.prefix(index); + TypeDescriptor parent = wrapper.getPropertyTypeDescriptor(name); + if (parent == null) { + return; + } + TypeDescriptor descriptor = parent.getMapValueTypeDescriptor(); + if (descriptor == null) { + descriptor = TypeDescriptor.valueOf(Object.class); + } + if (!descriptor.isMap() && !descriptor.isCollection() && !descriptor.getType().equals(Object.class)) { + return; + } + String extensionName = path.prefix(index + 1); + if (wrapper.isReadableProperty(extensionName)) { + Object currentValue = wrapper.getPropertyValue(extensionName); + if ((descriptor.isCollection() && currentValue instanceof Collection) + || (!descriptor.isCollection() && currentValue instanceof Map)) { + return; + } + } + Object extend = new LinkedHashMap(); + if (descriptor.isCollection()) { + extend = new ArrayList(); + } + if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) { + extend = BLANK; + } + wrapper.setPropertyValue(extensionName, extend); + } + + private String getActualPropertyName(BeanWrapper target, String prefix, String name) { + String propertyName = resolvePropertyName(target, prefix, name); + if (propertyName == null) { + propertyName = resolveNestedPropertyName(target, prefix, name); + } + return (propertyName != null ? propertyName : name); + } + + private String resolveNestedPropertyName(BeanWrapper target, String prefix, String name) { + StringBuilder candidate = new StringBuilder(); + for (String field : name.split("[_\\-\\.]")) { + candidate.append(candidate.length() > 0 ? "." : ""); + candidate.append(field); + String nested = resolvePropertyName(target, prefix, candidate.toString()); + if (nested != null) { + Class type = target.getPropertyType(nested); + if ((type != null) && Map.class.isAssignableFrom(type)) { + // Special case for map property (gh-3836). + return nested + "[" + name.substring(candidate.length() + 1) + "]"; + } + String propertyName = resolvePropertyName(target, + joinString(prefix, nested), + name.substring(candidate.length() + 1)); + if (propertyName != null) { + return joinString(nested, propertyName); + } + } + } + return null; + } + + private String resolvePropertyName(BeanWrapper target, String prefix, String name) { + Iterable names = getNameAndAliases(name); + for (String nameOrAlias : names) { + for (String candidate : new RelaxedNames(nameOrAlias)) { + try { + if (target.getPropertyType(joinString(prefix, candidate)) != null) { + return candidate; + } + } catch (InvalidPropertyException ex) { + // swallow and continue + } + } + } + return null; + } + + private String joinString(String prefix, String name) { + return (StringUtils.hasLength(prefix) ? prefix + "." + name : name); + } + + private Iterable getNameAndAliases(String name) { + List aliases = this.nameAliases.get(name); + if (aliases == null) { + return Collections.singleton(name); + } + List nameAndAliases = new ArrayList(aliases.size() + 1); + nameAndAliases.add(name); + nameAndAliases.addAll(aliases); + return nameAndAliases; + } + + private static Object wrapTarget(Object target) { + if (target instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) target; + target = new RelaxedDataBinder.MapHolder(map); + } + return target; + } + + @Override + public void registerCustomEditor(Class requiredType, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, propertyEditor); + } + } + + @Override + public void registerCustomEditor(Class requiredType, String field, PropertyEditor propertyEditor) { + if (propertyEditor == null || !EXCLUDED_EDITORS.contains(propertyEditor.getClass())) { + super.registerCustomEditor(requiredType, field, propertyEditor); + } + } + + /** + * Holder to allow Map targets to be bound. + */ + static class MapHolder { + + private Map map; + + MapHolder(Map map){ + this.map = map; + } + + public void setMap(Map map) { + this.map = map; + } + + public Map getMap() { + return this.map; + } + + } + + /** + * A path though properties of a bean. + */ + private static class BeanPath { + + private List nodes; + + BeanPath(String path){ + this.nodes = splitPath(path); + } + + public List prefixes() { + List prefixes = new ArrayList(); + for (int index = 1; index < this.nodes.size(); index++) { + prefixes.add(prefix(index)); + } + return prefixes; + } + + public boolean isLastNode(int index) { + return index >= this.nodes.size() - 1; + } + + private List splitPath(String path) { + List nodes = new ArrayList(); + String current = extractIndexedPaths(path, nodes); + for (String name : StringUtils.delimitedListToStringArray(current, ".")) { + if (StringUtils.hasText(name)) { + nodes.add(new RelaxedDataBinder.BeanPath.PropertyNode(name)); + } + } + return nodes; + } + + private String extractIndexedPaths(String path, List nodes) { + int startRef = path.indexOf("["); + String current = path; + while (startRef >= 0) { + if (startRef > 0) { + nodes.addAll(splitPath(current.substring(0, startRef))); + } + int endRef = current.indexOf("]", startRef); + if (endRef > 0) { + String sub = current.substring(startRef + 1, endRef); + if (sub.matches("[0-9]+")) { + nodes.add(new RelaxedDataBinder.BeanPath.ArrayIndexNode(sub)); + } else { + nodes.add(new RelaxedDataBinder.BeanPath.MapIndexNode(sub)); + } + } + current = current.substring(endRef + 1); + startRef = current.indexOf("["); + } + return current; + } + + public void collapseKeys(int index) { + List revised = new ArrayList(); + for (int i = 0; i < index; i++) { + revised.add(this.nodes.get(i)); + } + StringBuilder builder = new StringBuilder(); + for (int i = index; i < this.nodes.size(); i++) { + if (i > index) { + builder.append("."); + } + builder.append(this.nodes.get(i).name); + } + revised.add(new RelaxedDataBinder.BeanPath.PropertyNode(builder.toString())); + this.nodes = revised; + } + + public void mapIndex(int index) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(index); + if (node instanceof RelaxedDataBinder.BeanPath.PropertyNode) { + node = ((RelaxedDataBinder.BeanPath.PropertyNode) node).mapIndex(); + } + this.nodes.set(index, node); + } + + public String prefix(int index) { + return range(0, index); + } + + public void rename(int index, String name) { + this.nodes.get(index).name = name; + } + + public String name(int index) { + if (index < this.nodes.size()) { + return this.nodes.get(index).name; + } + return null; + } + + private String range(int start, int end) { + StringBuilder builder = new StringBuilder(); + for (int i = start; i < end; i++) { + RelaxedDataBinder.BeanPath.PathNode node = this.nodes.get(i); + builder.append(node); + } + if (builder.toString().startsWith(("."))) { + builder.replace(0, 1, ""); + } + return builder.toString(); + } + + public boolean isArrayIndex(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.ArrayIndexNode; + } + + public boolean isProperty(int index) { + return this.nodes.get(index) instanceof RelaxedDataBinder.BeanPath.PropertyNode; + } + + @Override + public String toString() { + return prefix(this.nodes.size()); + } + + private static class PathNode { + + protected String name; + + PathNode(String name){ + this.name = name; + } + + } + + private static class ArrayIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + ArrayIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class MapIndexNode extends RelaxedDataBinder.BeanPath.PathNode { + + MapIndexNode(String name){ + super(name); + } + + @Override + public String toString() { + return "[" + this.name + "]"; + } + + } + + private static class PropertyNode extends RelaxedDataBinder.BeanPath.PathNode { + + PropertyNode(String name){ + super(name); + } + + public RelaxedDataBinder.BeanPath.MapIndexNode mapIndex() { + return new RelaxedDataBinder.BeanPath.MapIndexNode(this.name); + } + + @Override + public String toString() { + return "." + this.name; + } + + } + + } + + /** + * Extended version of {@link BeanPropertyBindingResult} to support relaxed + * binding. + */ + private static class RelaxedBeanPropertyBindingResult extends BeanPropertyBindingResult { + + private RelaxedConversionService conversionService; + + RelaxedBeanPropertyBindingResult(Object target, String objectName, boolean autoGrowNestedPaths, + int autoGrowCollectionLimit, ConversionService conversionService){ + super(target, objectName, autoGrowNestedPaths, autoGrowCollectionLimit); + this.conversionService = new RelaxedConversionService(conversionService); + } + + @Override + protected BeanWrapper createBeanWrapper() { + BeanWrapper beanWrapper = new RelaxedDataBinder.RelaxedBeanWrapper(getTarget()); + beanWrapper.setConversionService(this.conversionService); + beanWrapper.registerCustomEditor(InetAddress.class, new InetAddressEditor()); + return beanWrapper; + } + + } + + /** + * Extended version of {@link BeanWrapperImpl} to support relaxed binding. + */ + private static class RelaxedBeanWrapper extends BeanWrapperImpl { + + private static final Set BENIGN_PROPERTY_SOURCE_NAMES; + + static { + Set names = new HashSet(); + names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); + names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); + BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names); + } + + RelaxedBeanWrapper(Object target){ + super(target); + } + + @Override + public void setPropertyValue(PropertyValue pv) throws BeansException { + try { + super.setPropertyValue(pv); + } catch (NotWritablePropertyException ex) { + PropertyOrigin origin = OriginCapablePropertyValue.getOrigin(pv); + if (isBenign(origin)) { + logger.debug("Ignoring benign property binding failure", ex); + return; + } + if (origin == null) { + throw ex; + } + throw new RelaxedBindingNotWritablePropertyException(ex, origin); + } + } + + private boolean isBenign(PropertyOrigin origin) { + String name = (origin != null ? origin.getSource().getName() : null); + return BENIGN_PROPERTY_SOURCE_NAMES.contains(name); + } + + } + + public static class RelaxedBindingNotWritablePropertyException extends NotWritablePropertyException { + + private final String message; + + private final PropertyOrigin propertyOrigin; + + RelaxedBindingNotWritablePropertyException(NotWritablePropertyException ex, PropertyOrigin propertyOrigin){ + super(ex.getBeanClass(), ex.getPropertyName()); + this.propertyOrigin = propertyOrigin; + this.message = "Failed to bind '" + propertyOrigin.getName() + "' from '" + + propertyOrigin.getSource().getName() + "' to '" + ex.getPropertyName() + "' property on '" + + ex.getBeanClass().getName() + "'"; + } + + @Override + public String getMessage() { + return this.message; + } + + public PropertyOrigin getPropertyOrigin() { + return this.propertyOrigin; + } + + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java new file mode 100644 index 00000000..f1b1628f --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java @@ -0,0 +1,241 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.util.StringUtils; + +/** + * Generates relaxed name variations from a given source. + * + * @author Phillip Webb + * @author Dave Syer + * @see RelaxedDataBinder + */ +public final class RelaxedNames implements Iterable { + + private static final Pattern CAMEL_CASE_PATTERN = Pattern.compile("([^A-Z-])([A-Z])"); + + private static final Pattern SEPARATED_TO_CAMEL_CASE_PATTERN = Pattern.compile("[_\\-.]"); + + private final String name; + + private final Set values = new LinkedHashSet(); + + /** + * Create a new {@link RelaxedNames} instance. + * + * @param name the source name. For the maximum number of variations specify the + * name using dashed notation (e.g. {@literal my-property-name} + */ + public RelaxedNames(String name){ + this.name = (name != null ? name : ""); + initialize(RelaxedNames.this.name, this.values); + } + + @Override + public Iterator iterator() { + return this.values.iterator(); + } + + private void initialize(String name, Set values) { + if (values.contains(name)) { + return; + } + for (RelaxedNames.Variation variation : RelaxedNames.Variation.values()) { + for (RelaxedNames.Manipulation manipulation : RelaxedNames.Manipulation.values()) { + String result = name; + result = manipulation.apply(result); + result = variation.apply(result); + values.add(result); + initialize(result, values); + } + } + } + + /** + * Name variations. + */ + enum Variation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + LOWERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toLowerCase(Locale.ENGLISH)); + } + + }, + + UPPERCASE { + + @Override + public String apply(String value) { + return (value.isEmpty() ? value : value.toUpperCase(Locale.ENGLISH)); + } + + }; + + public abstract String apply(String value); + + } + + /** + * Name manipulations. + */ + enum Manipulation { + + NONE { + + @Override + public String apply(String value) { + return value; + } + + }, + + HYPHEN_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('-') != -1 ? value.replace('-', '_') : value); + } + + }, + + UNDERSCORE_TO_PERIOD { + + @Override + public String apply(String value) { + return (value.indexOf('_') != -1 ? value.replace('_', '.') : value); + } + + }, + + PERIOD_TO_UNDERSCORE { + + @Override + public String apply(String value) { + return (value.indexOf('.') != -1 ? value.replace('.', '_') : value); + } + + }, + + CAMELCASE_TO_UNDERSCORE { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '_' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + CAMELCASE_TO_HYPHEN { + + @Override + public String apply(String value) { + if (value.isEmpty()) { + return value; + } + Matcher matcher = CAMEL_CASE_PATTERN.matcher(value); + if (!matcher.find()) { + return value; + } + matcher = matcher.reset(); + StringBuffer result = new StringBuffer(); + while (matcher.find()) { + matcher.appendReplacement(result, + matcher.group(1) + '-' + StringUtils.uncapitalize(matcher.group(2))); + } + matcher.appendTail(result); + return result.toString(); + } + + }, + + SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, false); + } + + }, + + CASE_INSENSITIVE_SEPARATED_TO_CAMELCASE { + + @Override + public String apply(String value) { + return separatedToCamelCase(value, true); + } + + }; + + private static final char[] SUFFIXES = new char[] { '_', '-', '.' }; + + public abstract String apply(String value); + + private static String separatedToCamelCase(String value, boolean caseInsensitive) { + if (value.isEmpty()) { + return value; + } + StringBuilder builder = new StringBuilder(); + for (String field : SEPARATED_TO_CAMEL_CASE_PATTERN.split(value)) { + field = (caseInsensitive ? field.toLowerCase(Locale.ENGLISH) : field); + builder.append(builder.length() != 0 ? StringUtils.capitalize(field) : field); + } + char lastChar = value.charAt(value.length() - 1); + for (char suffix : SUFFIXES) { + if (lastChar == suffix) { + builder.append(suffix); + break; + } + } + return builder.toString(); + } + + } + + /** + * Return a {@link RelaxedNames} for the given source camelCase source name. + * + * @param name the source name in camelCase + * @return the relaxed names + */ + public static RelaxedNames forCamelCase(String name) { + StringBuilder result = new StringBuilder(); + for (char c : name.toCharArray()) { + result.append(Character.isUpperCase(c) && result.length() > 0 + && result.charAt(result.length() - 1) != '-' ? "-" + Character.toLowerCase(c) : c); + } + return new RelaxedNames(result.toString()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java new file mode 100644 index 00000000..00a2d808 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/StringToCharArrayConverter.java @@ -0,0 +1,17 @@ +package com.alibaba.otter.canal.client.adapter.config.bind; + +import org.springframework.core.convert.converter.Converter; + +/** + * Converts a String to a Char Array. + * + * @author Phillip Webb + */ +class StringToCharArrayConverter implements Converter { + + @Override + public char[] convert(String source) { + return source.toCharArray(); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java new file mode 100644 index 00000000..9cb73e90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/AbstractResource.java @@ -0,0 +1,203 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +import org.springframework.util.Assert; +import org.springframework.util.ResourceUtils; + +/** + * Convenience base class for {@link Resource} implementations, pre-implementing + * typical behavior. + *

+ * The "exists" method will check whether a File or InputStream can be opened; + * "isOpen" will always return false; "getURL" and "getFile" throw an exception; + * and "toString" will return the description. + * + * @author Juergen Hoeller + * @since 28.12.2003 + */ +public abstract class AbstractResource implements Resource { + + /** + * This implementation checks whether a File can be opened, falling back to + * whether an InputStream can be opened. This will cover both directories and + * content resources. + */ + @Override + public boolean exists() { + // Try file existence: can we find the file in the file system? + try { + return getFile().exists(); + } catch (IOException ex) { + // Fall back to stream existence: can we open the stream? + try { + InputStream is = getInputStream(); + is.close(); + return true; + } catch (Throwable isEx) { + return false; + } + } + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean isReadable() { + return true; + } + + /** + * This implementation always returns {@code false}. + */ + @Override + public boolean isOpen() { + return false; + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to a URL. + */ + @Override + public URL getURL() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to URL"); + } + + /** + * This implementation builds a URI based on the URL returned by + * {@link #getURL()}. + */ + @Override + public URI getURI() throws IOException { + URL url = getURL(); + try { + return ResourceUtils.toURI(url); + } catch (URISyntaxException ex) { + throw new RuntimeException("Invalid URI [" + url + "]", ex); + } + } + + /** + * This implementation throws a FileNotFoundException, assuming that the + * resource cannot be resolved to an absolute file path. + */ + @Override + public File getFile() throws IOException { + throw new FileNotFoundException(getDescription() + " cannot be resolved to absolute file path"); + } + + /** + * This implementation reads the entire InputStream to calculate the content + * length. Subclasses will almost always be able to provide a more optimal + * version of this, e.g. checking a File length. + * + * @see #getInputStream() + */ + @Override + public long contentLength() throws IOException { + InputStream is = getInputStream(); + Assert.state(is != null, "Resource InputStream must not be null"); + try { + long size = 0; + byte[] buf = new byte[255]; + int read; + while ((read = is.read(buf)) != -1) { + size += read; + } + return size; + } finally { + try { + is.close(); + } catch (IOException ex) { + } + } + } + + /** + * This implementation checks the timestamp of the underlying File, if + * available. + * + * @see #getFileForLastModifiedCheck() + */ + @Override + public long lastModified() throws IOException { + long lastModified = getFileForLastModifiedCheck().lastModified(); + if (lastModified == 0L) { + throw new FileNotFoundException( + getDescription() + " cannot be resolved in the file system for resolving its last-modified timestamp"); + } + return lastModified; + } + + /** + * Determine the File to use for timestamp checking. + *

+ * The default implementation delegates to {@link #getFile()}. + * + * @return the File to use for timestamp checking (never {@code null}) + * @throws FileNotFoundException if the resource cannot be resolved as an + * absolute file path, i.e. is not available in a file system + * @throws IOException in case of general resolution/reading failures + */ + protected File getFileForLastModifiedCheck() throws IOException { + return getFile(); + } + + /** + * This implementation throws a FileNotFoundException, assuming that relative + * resources cannot be created for this resource. + */ + @Override + public org.springframework.core.io.Resource createRelative(String relativePath) throws IOException { + throw new FileNotFoundException("Cannot create a relative resource for " + getDescription()); + } + + /** + * This implementation always returns {@code null}, assuming that this resource + * type does not have a filename. + */ + @Override + public String getFilename() { + return null; + } + + /** + * This implementation returns the description of this resource. + * + * @see #getDescription() + */ + @Override + public String toString() { + return getDescription(); + } + + /** + * This implementation compares description strings. + * + * @see #getDescription() + */ + @Override + public boolean equals(Object obj) { + return (obj == this + || (obj instanceof org.springframework.core.io.Resource + && ((org.springframework.core.io.Resource) obj).getDescription().equals(getDescription()))); + } + + /** + * This implementation returns the description's hash code. + * + * @see #getDescription() + */ + @Override + public int hashCode() { + return getDescription().hashCode(); + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java new file mode 100644 index 00000000..3ee85766 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/ByteArrayResource.java @@ -0,0 +1,118 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.InputStreamSource; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * {@link Resource} implementation for a given byte array. + *

+ * Creates a {@link ByteArrayInputStream} for the given byte array. + *

+ * Useful for loading content from any given byte array, without having to + * resort to a single-use {@link InputStreamResource}. Particularly useful for + * creating mail attachments from local content, where JavaMail needs to be able + * to read the stream multiple times. + * + * @author Juergen Hoeller + * @author Sam Brannen + * @see ByteArrayInputStream + * @see InputStreamResource + * @since 1.2.3 + */ +public class ByteArrayResource extends AbstractResource { + + private final byte[] byteArray; + + private final String description; + + /** + * Create a new {@code ByteArrayResource}. + * + * @param byteArray the byte array to wrap + */ + public ByteArrayResource(byte[] byteArray){ + this(byteArray, "resource loaded from byte array"); + } + + /** + * Create a new {@code ByteArrayResource} with a description. + * + * @param byteArray the byte array to wrap + * @param description where the byte array comes from + */ + public ByteArrayResource(byte[] byteArray, String description){ + Assert.notNull(byteArray, "Byte array must not be null"); + this.byteArray = byteArray; + this.description = (description != null ? description : ""); + } + + /** + * Return the underlying byte array. + */ + public final byte[] getByteArray() { + return this.byteArray; + } + + /** + * This implementation always returns {@code true}. + */ + @Override + public boolean exists() { + return true; + } + + /** + * This implementation returns the length of the underlying byte array. + */ + @Override + public long contentLength() { + return this.byteArray.length; + } + + /** + * This implementation returns a ByteArrayInputStream for the underlying byte + * array. + * + * @see ByteArrayInputStream + */ + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(this.byteArray); + } + + /** + * This implementation returns a description that includes the passed-in + * {@code description}, if any. + */ + @Override + public String getDescription() { + return "Byte array resource [" + this.description + "]"; + } + + /** + * This implementation compares the underlying byte array. + * + * @see Arrays#equals(byte[], byte[]) + */ + @Override + public boolean equals(Object obj) { + return (obj == this || (obj instanceof org.springframework.core.io.ByteArrayResource + && Arrays.equals(((ByteArrayResource) obj).byteArray, this.byteArray))); + } + + /** + * This implementation returns the hash code based on the underlying byte array. + */ + @Override + public int hashCode() { + return (byte[].class.hashCode() * 29 * this.byteArray.length); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java new file mode 100644 index 00000000..106f8f90 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java @@ -0,0 +1,107 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.util.StringUtils; + +/** + * Composite {@link PropertySource} implementation that iterates over a set of + * {@link PropertySource} instances. Necessary in cases where multiple property + * sources share the same name, e.g. when multiple values are supplied to + * {@code @PropertySource}. + *

+ * As of Spring 4.1.2, this class extends {@link EnumerablePropertySource} + * instead of plain {@link PropertySource}, exposing {@link #getPropertyNames()} + * based on the accumulated property names from all contained sources (as far as + * possible). + * + * @author Chris Beams + * @author Juergen Hoeller + * @author Phillip Webb + * @since 3.1.1 + */ +public class CompositePropertySource extends EnumerablePropertySource { + + private final Set> propertySources = new LinkedHashSet>(); + + /** + * Create a new {@code CompositePropertySource}. + * + * @param name the name of the property source + */ + public CompositePropertySource(String name){ + super(name); + } + + @Override + public Object getProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + Object candidate = propertySource.getProperty(name); + if (candidate != null) { + return candidate; + } + } + return null; + } + + @Override + public boolean containsProperty(String name) { + for (PropertySource propertySource : this.propertySources) { + if (propertySource.containsProperty(name)) { + return true; + } + } + return false; + } + + @Override + public String[] getPropertyNames() { + Set names = new LinkedHashSet(); + for (PropertySource propertySource : this.propertySources) { + if (!(propertySource instanceof EnumerablePropertySource)) { + throw new IllegalStateException( + "Failed to enumerate property names due to non-enumerable property source: " + propertySource); + } + names.addAll(Arrays.asList(((EnumerablePropertySource) propertySource).getPropertyNames())); + } + return StringUtils.toStringArray(names); + } + + /** + * Add the given {@link PropertySource} to the end of the chain. + * + * @param propertySource the PropertySource to add + */ + public void addPropertySource(PropertySource propertySource) { + this.propertySources.add(propertySource); + } + + /** + * Add the given {@link PropertySource} to the start of the chain. + * + * @param propertySource the PropertySource to add + * @since 4.1 + */ + public void addFirstPropertySource(PropertySource propertySource) { + List> existing = new ArrayList>(this.propertySources); + this.propertySources.clear(); + this.propertySources.add(propertySource); + this.propertySources.addAll(existing); + } + + /** + * Return all property sources that this composite source holds. + * + * @since 4.1.1 + */ + public Collection> getPropertySources() { + return this.propertySources; + } + + @Override + public String toString() { + return String + .format("%s [name='%s', propertySources=%s]", getClass().getSimpleName(), this.name, this.propertySources); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java new file mode 100644 index 00000000..e5ce8639 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/EnumerablePropertySource.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.springframework.util.ObjectUtils; + +/** + * A {@link PropertySource} implementation capable of interrogating its + * underlying source object to enumerate all possible property name/value pairs. + * Exposes the {@link #getPropertyNames()} method to allow callers to introspect + * available properties without having to access the underlying source object. + * This also facilitates a more efficient implementation of + * {@link #containsProperty(String)}, in that it can call + * {@link #getPropertyNames()} and iterate through the returned array rather + * than attempting a call to {@link #getProperty(String)} which may be more + * expensive. Implementations may consider caching the result of + * {@link #getPropertyNames()} to fully exploit this performance opportunity. + *

+ * Most framework-provided {@code PropertySource} implementations are + * enumerable; a counter-example would be {@code JndiPropertySource} where, due + * to the nature of JNDI it is not possible to determine all possible property + * names at any given time; rather it is only possible to try to access a + * property (via {@link #getProperty(String)}) in order to evaluate whether it + * is present or not. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public abstract class EnumerablePropertySource extends PropertySource { + + public EnumerablePropertySource(String name, T source){ + super(name, source); + } + + protected EnumerablePropertySource(String name){ + super(name); + } + + /** + * Return whether this {@code PropertySource} contains a property with the given + * name. + *

+ * This implementation checks for the presence of the given name within the + * {@link #getPropertyNames()} array. + * + * @param name the name of the property to find + */ + @Override + public boolean containsProperty(String name) { + return ObjectUtils.containsElement(getPropertyNames(), name); + } + + /** + * Return the names of all properties contained by the {@linkplain #getSource() + * source} object (never {@code null}). + */ + public abstract String[] getPropertyNames(); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java new file mode 100644 index 00000000..eecc0ee4 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MapPropertySource.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; + +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; +import org.springframework.util.StringUtils; + +/** + * {@link PropertySource} that reads keys and values from a {@code Map} object. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + * @see PropertiesPropertySource + */ +public class MapPropertySource extends EnumerablePropertySource> { + + public MapPropertySource(String name, Map source){ + super(name, source); + } + + @Override + public Object getProperty(String name) { + return this.source.get(name); + } + + @Override + public boolean containsProperty(String name) { + return this.source.containsKey(name); + } + + @Override + public String[] getPropertyNames() { + return StringUtils.toStringArray(this.source.keySet()); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java new file mode 100644 index 00000000..d8f32d3c --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java @@ -0,0 +1,221 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.core.env.PropertyResolver; +import org.springframework.core.env.PropertySourcesPropertyResolver; + +/** + * Default implementation of the {@link PropertySources} interface. Allows + * manipulation of contained property sources and provides a constructor for + * copying an existing {@code PropertySources} instance. + *

+ * Where precedence is mentioned in methods such as {@link #addFirst} + * and {@link #addLast}, this is with regard to the order in which property + * sources will be searched when resolving a given property with a + * {@link PropertyResolver}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @see PropertySourcesPropertyResolver + * @since 3.1 + */ +public class MutablePropertySources implements PropertySources { + + private final Log logger; + + private final List> propertySourceList = new CopyOnWriteArrayList>(); + + /** + * Create a new {@link MutablePropertySources} + * object. + */ + public MutablePropertySources(){ + this.logger = LogFactory.getLog(getClass()); + } + + /** + * Create a new {@code MutablePropertySources} from the given propertySources + * object, preserving the original order of contained {@code PropertySource} + * objects. + */ + public MutablePropertySources(PropertySources propertySources){ + this(); + for (PropertySource propertySource : propertySources) { + addLast(propertySource); + } + } + + /** + * Create a new {@link MutablePropertySources} + * object and inherit the given logger, usually from an enclosing + * {@link Environment}. + */ + MutablePropertySources(Log logger){ + this.logger = logger; + } + + @Override + public boolean contains(String name) { + return this.propertySourceList.contains(PropertySource.named(name)); + } + + @Override + public PropertySource get(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.get(index) : null); + } + + @Override + public Iterator> iterator() { + return this.propertySourceList.iterator(); + } + + /** + * Add the given property source object with highest precedence. + */ + public void addFirst(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with highest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(0, propertySource); + } + + /** + * Add the given property source object with lowest precedence. + */ + public void addLast(PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + "' with lowest search precedence"); + } + removeIfPresent(propertySource); + this.propertySourceList.add(propertySource); + } + + /** + * Add the given property source object with precedence immediately higher than + * the named relative property source. + */ + public void addBefore(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately higher than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index, propertySource); + } + + /** + * Add the given property source object with precedence immediately lower than + * the named relative property source. + */ + public void addAfter(String relativePropertySourceName, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Adding PropertySource '" + propertySource.getName() + + "' with search precedence immediately lower than '" + relativePropertySourceName + "'"); + } + assertLegalRelativeAddition(relativePropertySourceName, propertySource); + removeIfPresent(propertySource); + int index = assertPresentAndGetIndex(relativePropertySourceName); + addAtIndex(index + 1, propertySource); + } + + /** + * Return the precedence of the given property source, {@code -1} if not found. + */ + public int precedenceOf(PropertySource propertySource) { + return this.propertySourceList.indexOf(propertySource); + } + + /** + * Remove and return the property source with the given name, {@code null} if + * not found. + * + * @param name the name of the property source to find and remove + */ + public PropertySource remove(String name) { + if (logger.isDebugEnabled()) { + logger.debug("Removing PropertySource '" + name + "'"); + } + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + return (index != -1 ? this.propertySourceList.remove(index) : null); + } + + /** + * Replace the property source with the given name with the given property + * source object. + * + * @param name the name of the property source to find and replace + * @param propertySource the replacement property source + * @throws IllegalArgumentException if no property source with the given name is + * present + * @see #contains + */ + public void replace(String name, PropertySource propertySource) { + if (logger.isDebugEnabled()) { + logger.debug("Replacing PropertySource '" + name + "' with '" + propertySource.getName() + "'"); + } + int index = assertPresentAndGetIndex(name); + this.propertySourceList.set(index, propertySource); + } + + /** + * Return the number of {@link PropertySource} objects contained. + */ + public int size() { + return this.propertySourceList.size(); + } + + @Override + public String toString() { + return this.propertySourceList.toString(); + } + + /** + * Ensure that the given property source is not being added relative to itself. + */ + protected void assertLegalRelativeAddition(String relativePropertySourceName, PropertySource propertySource) { + String newPropertySourceName = propertySource.getName(); + if (relativePropertySourceName.equals(newPropertySourceName)) { + throw new IllegalArgumentException( + "PropertySource named '" + newPropertySourceName + "' cannot be added relative to itself"); + } + } + + /** + * Remove the given property source if it is present. + */ + protected void removeIfPresent(PropertySource propertySource) { + this.propertySourceList.remove(propertySource); + } + + /** + * Add the given property source at a particular index in the list. + */ + private void addAtIndex(int index, PropertySource propertySource) { + removeIfPresent(propertySource); + this.propertySourceList.add(index, propertySource); + } + + /** + * Assert that the named property source is present and return its index. + * + * @param name {@linkplain PropertySource#getName() name of the property source} + * to find + * @throws IllegalArgumentException if the named property source is not present + */ + private int assertPresentAndGetIndex(String name) { + int index = this.propertySourceList.indexOf(PropertySource.named(name)); + if (index == -1) { + throw new IllegalArgumentException("PropertySource named '" + name + "' does not exist"); + } + return index; + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java new file mode 100644 index 00000000..12fe686a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertiesPropertySource.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.Map; +import java.util.Properties; + +import org.springframework.core.env.PropertySource; + +/** + * {@link PropertySource} implementation that extracts properties from a + * {@link Properties} object. + *

+ * Note that because a {@code Properties} object is technically an + * {@code } {@link java.util.Hashtable Hashtable}, one may + * contain non-{@code String} keys or values. This implementation, however is + * restricted to accessing only {@code String}-based keys and values, in the + * same fashion as {@link Properties#getProperty} and + * {@link Properties#setProperty}. + * + * @author Chris Beams + * @author Juergen Hoeller + * @since 3.1 + */ +public class PropertiesPropertySource extends MapPropertySource { + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public PropertiesPropertySource(String name, Properties source){ + super(name, (Map) source); + } + + protected PropertiesPropertySource(String name, Map source){ + super(name, source); + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java new file mode 100644 index 00000000..f7e4cc34 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySource.java @@ -0,0 +1,239 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * Abstract base class representing a source of name/value property pairs. The + * underlying {@linkplain #getSource() source object} may be of any type + * {@code T} that encapsulates properties. Examples include + * {@link java.util.Properties} objects, {@link java.util.Map} objects, + * {@code ServletContext} and {@code ServletConfig} objects (for access to init + * parameters). Explore the {@code PropertySource} type hierarchy to see + * provided implementations. + *

+ * {@code PropertySource} objects are not typically used in isolation, but + * rather through a {@link PropertySources} object, which aggregates property + * sources and in conjunction with a {@link PropertyResolver} implementation + * that can perform precedence-based searches across the set of + * {@code PropertySources}. + *

+ * {@code PropertySource} identity is determined not based on the content of + * encapsulated properties, but rather based on the {@link #getName() name} of + * the {@code PropertySource} alone. This is useful for manipulating + * {@code PropertySource} objects when in collection contexts. See operations in + * {@link MutablePropertySources} as well as the {@link #named(String)} and + * {@link #toString()} methods for details. + *

+ * Note that when working + * with @{@link org.springframework.context.annotation.Configuration + * Configuration} classes that the @{@link PropertySource PropertySource} + * annotation provides a convenient and declarative way of adding property + * sources to the enclosing {@code Environment}. + * + * @author Chris Beams + * @since 3.1 + * @see PropertySources + * @see MutablePropertySources + * @see PropertySource + */ +public abstract class PropertySource { + + protected final Log logger = LogFactory.getLog(getClass()); + + protected final String name; + + protected final T source; + + /** + * Create a new {@code PropertySource} with the given name and source object. + */ + public PropertySource(String name, T source){ + Assert.hasText(name, "Property source name must contain at least one character"); + Assert.notNull(source, "Property source must not be null"); + this.name = name; + this.source = source; + } + + /** + * Create a new {@code PropertySource} with the given name and with a new + * {@code Object} instance as the underlying source. + *

+ * Often useful in testing scenarios when creating anonymous implementations + * that never query an actual source but rather return hard-coded values. + */ + @SuppressWarnings("unchecked") + public PropertySource(String name){ + this(name, (T) new Object()); + } + + /** + * Return the name of this {@code PropertySource} + */ + public String getName() { + return this.name; + } + + /** + * Return the underlying source object for this {@code PropertySource}. + */ + public T getSource() { + return this.source; + } + + /** + * Return whether this {@code PropertySource} contains the given name. + *

+ * This implementation simply checks for a {@code null} return value from + * {@link #getProperty(String)}. Subclasses may wish to implement a more + * efficient algorithm if possible. + * + * @param name the property name to find + */ + public boolean containsProperty(String name) { + return (getProperty(name) != null); + } + + /** + * Return the value associated with the given name, or {@code null} if not + * found. + * + * @param name the property to find + */ + public abstract Object getProperty(String name); + + /** + * This {@code PropertySource} object is equal to the given object if: + *

    + *
  • they are the same instance + *
  • the {@code name} properties for both objects are equal + *
+ *

+ * No properties other than {@code name} are evaluated. + */ + @Override + public boolean equals(Object obj) { + return (this == obj || (obj instanceof PropertySource + && ObjectUtils.nullSafeEquals(this.name, ((PropertySource) obj).name))); + } + + /** + * Return a hash code derived from the {@code name} property of this + * {@code PropertySource} object. + */ + @Override + public int hashCode() { + return ObjectUtils.nullSafeHashCode(this.name); + } + + /** + * Produce concise output (type and name) if the current log level does not + * include debug. If debug is enabled, produce verbose output including the hash + * code of the PropertySource instance and every name/value property pair. + *

+ * This variable verbosity is useful as a property source such as system + * properties or environment variables may contain an arbitrary number of + * property pairs, potentially leading to difficult to read exception and log + * messages. + * + * @see Log#isDebugEnabled() + */ + @Override + public String toString() { + if (logger.isDebugEnabled()) { + return getClass().getSimpleName() + "@" + System.identityHashCode(this) + " {name='" + this.name + + "', properties=" + this.source + "}"; + } else { + return getClass().getSimpleName() + " {name='" + this.name + "'}"; + } + } + + /** + * Return a {@code PropertySource} implementation intended for collection + * comparison purposes only. + *

+ * Primarily for internal use, but given a collection of {@code PropertySource} + * objects, may be used as follows: + * + *

+     *
+     * {
+     *     @code
+     *     List> sources = new ArrayList>();
+     *     sources.add(new MapPropertySource("sourceA", mapA));
+     *     sources.add(new MapPropertySource("sourceB", mapB));
+     *     assert sources.contains(PropertySource.named("sourceA"));
+     *     assert sources.contains(PropertySource.named("sourceB"));
+     *     assert !sources.contains(PropertySource.named("sourceC"));
+     * }
+     * 
+ * + * The returned {@code PropertySource} will throw + * {@code UnsupportedOperationException} if any methods other than + * {@code equals(Object)}, {@code hashCode()}, and {@code toString()} are + * called. + * + * @param name the name of the comparison {@code PropertySource} to be created + * and returned. + */ + public static PropertySource named(String name) { + return new ComparisonPropertySource(name); + } + + /** + * {@code PropertySource} to be used as a placeholder in cases where an actual + * property source cannot be eagerly initialized at application context creation + * time. For example, a {@code ServletContext}-based property source must wait + * until the {@code ServletContext} object is available to its enclosing + * {@code ApplicationContext}. In such cases, a stub should be used to hold the + * intended default position/order of the property source, then be replaced + * during context refresh. + * + * @see org.springframework.web.context.support.StandardServletEnvironment + * @see org.springframework.web.context.support.ServletContextPropertySource + */ + public static class StubPropertySource extends PropertySource { + + public StubPropertySource(String name){ + super(name, new Object()); + } + + /** + * Always returns {@code null}. + */ + @Override + public String getProperty(String name) { + return null; + } + } + + /** + * @see PropertySource#named(String) + */ + static class ComparisonPropertySource extends StubPropertySource { + + private static final String USAGE_ERROR = "ComparisonPropertySource instances are for use with collection comparison only"; + + public ComparisonPropertySource(String name){ + super(name); + } + + @Override + public Object getSource() { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public boolean containsProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + + @Override + public String getProperty(String name) { + throw new UnsupportedOperationException(USAGE_ERROR); + } + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java new file mode 100644 index 00000000..982f61b8 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySourceLoader.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; + +import org.springframework.core.io.support.SpringFactoriesLoader; + +/** + * Strategy interface located via {@link SpringFactoriesLoader} and used to load + * a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + */ +public interface PropertySourceLoader { + + /** + * Returns the file extensions that the loader supports (excluding the '.'). + * + * @return the file extensions + */ + String[] getFileExtensions(); + + /** + * Load the resource into a property source. + * + * @param name the name of the property source + * @param resource the resource to load + * @param profile the name of the profile to load or {@code null}. The profile + * can be used to load multi-document files (such as YAML). Simple property + * formats should {@code null} when asked to load a profile. + * @return a property source or {@code null} + * @throws IOException if the source cannot be loaded + */ + PropertySource load(String name, Resource resource, String profile) throws IOException; +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java new file mode 100644 index 00000000..3cd08c9a --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/PropertySources.java @@ -0,0 +1,25 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +/** + * Holder containing one or more {@link PropertySource} objects. + * + * @author Chris Beams + * @since 3.1 + */ +public interface PropertySources extends Iterable> { + + /** + * Return whether a property source with the given name is contained. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + boolean contains(String name); + + /** + * Return the property source with the given name, {@code null} if not found. + * + * @param name the {@linkplain PropertySource#getName() name of the property source} to find + */ + PropertySource get(String name); + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java new file mode 100644 index 00000000..88301f20 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/Resource.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URL; + +import org.springframework.core.io.*; +import org.springframework.core.io.ByteArrayResource; + +/** + * Interface for a resource descriptor that abstracts from the actual type of + * underlying resource, such as a file or class path resource. + *

+ * An InputStream can be opened for every resource if it exists in physical + * form, but a URL or File handle can just be returned for certain resources. + * The actual behavior is implementation-specific. + * + * @author Juergen Hoeller + * @since 28.12.2003 + * @see #getInputStream() + * @see #getURL() + * @see #getURI() + * @see #getFile() + * @see WritableResource + * @see ContextResource + * @see UrlResource + * @see ClassPathResource + * @see FileSystemResource + * @see PathResource + * @see ByteArrayResource + * @see InputStreamResource + */ +public interface Resource extends InputStreamSource { + + boolean exists(); + + boolean isReadable(); + + boolean isOpen(); + + URL getURL() throws IOException; + + URI getURI() throws IOException; + + File getFile() throws IOException; + + long contentLength() throws IOException; + + long lastModified() throws IOException; + + org.springframework.core.io.Resource createRelative(String var1) throws IOException; + + String getFilename(); + + String getDescription(); +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java new file mode 100644 index 00000000..f59c6983 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java @@ -0,0 +1,182 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.util.*; + +import org.springframework.core.env.Environment; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * {@link YamlProcessor.DocumentMatcher} backed by + * {@link Environment#getActiveProfiles()}. A YAML document may define a + * "spring.profiles" element as a comma-separated list of Spring profile names, + * optionally negated using the {@code !} character. If both negated and + * non-negated profiles are specified for a single document, at least one + * non-negated profile must match and no negated profiles may match. + * + * @author Dave Syer + * @author Matt Benson + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatcher { + + private String[] activeProfiles = new String[0]; + + public SpringProfileDocumentMatcher(){ + } + + public SpringProfileDocumentMatcher(String... profiles){ + addActiveProfiles(profiles); + } + + public void addActiveProfiles(String... profiles) { + LinkedHashSet set = new LinkedHashSet(Arrays.asList(this.activeProfiles)); + Collections.addAll(set, profiles); + this.activeProfiles = set.toArray(new String[set.size()]); + } + + @Override + public YamlProcessor.MatchStatus matches(Properties properties) { + List profiles = extractSpringProfiles(properties); + ProfilesMatcher profilesMatcher = getProfilesMatcher(); + Set negative = extractProfiles(profiles, ProfileType.NEGATIVE); + Set positive = extractProfiles(profiles, ProfileType.POSITIVE); + if (!CollectionUtils.isEmpty(negative)) { + if (profilesMatcher.matches(negative) == YamlProcessor.MatchStatus.FOUND) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + if (CollectionUtils.isEmpty(positive)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return profilesMatcher.matches(positive); + } + + private List extractSpringProfiles(Properties properties) { + SpringProperties springProperties = new SpringProperties(); + MutablePropertySources propertySources = new MutablePropertySources(); + propertySources.addFirst(new PropertiesPropertySource("profiles", properties)); + // PropertyValues propertyValues = new PropertySourcesPropertyValues( + // propertySources); + // new RelaxedDataBinder(springProperties, "spring").bind(propertyValues); + // TODO + List profiles = springProperties.getProfiles(); + return profiles; + } + + private ProfilesMatcher getProfilesMatcher() { + return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher( + new HashSet(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher()); + } + + private Set extractProfiles(List profiles, ProfileType type) { + if (CollectionUtils.isEmpty(profiles)) { + return null; + } + Set extractedProfiles = new HashSet(); + for (String candidate : profiles) { + ProfileType candidateType = ProfileType.POSITIVE; + if (candidate.startsWith("!")) { + candidateType = ProfileType.NEGATIVE; + } + if (candidateType == type) { + extractedProfiles.add(type != ProfileType.POSITIVE ? candidate.substring(1) : candidate); + } + } + return extractedProfiles; + } + + /** + * Profile match types. + */ + enum ProfileType { + + POSITIVE, NEGATIVE + + } + + /** + * Base class for profile matchers. + */ + private abstract static class ProfilesMatcher { + + public final YamlProcessor.MatchStatus matches(Set profiles) { + if (CollectionUtils.isEmpty(profiles)) { + return YamlProcessor.MatchStatus.ABSTAIN; + } + return doMatches(profiles); + } + + protected abstract YamlProcessor.MatchStatus doMatches(Set profiles); + + } + + /** + * {@link ProfilesMatcher} that matches when a value in {@code spring.profiles} + * is also in {@code spring.profiles.active}. + */ + private static class ActiveProfilesMatcher extends ProfilesMatcher { + + private final Set activeProfiles; + + ActiveProfilesMatcher(Set activeProfiles){ + this.activeProfiles = activeProfiles; + } + + @Override + protected YamlProcessor.MatchStatus doMatches(Set profiles) { + if (profiles.isEmpty()) { + return YamlProcessor.MatchStatus.NOT_FOUND; + } + for (String activeProfile : this.activeProfiles) { + if (profiles.contains(activeProfile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * {@link ProfilesMatcher} that matches when {@code + * spring.profiles} is empty or contains a value with no text. + * + * @see StringUtils#hasText(String) + */ + private static class EmptyProfilesMatcher extends ProfilesMatcher { + + @Override + public YamlProcessor.MatchStatus doMatches(Set springProfiles) { + if (springProfiles.isEmpty()) { + return YamlProcessor.MatchStatus.FOUND; + } + for (String profile : springProfiles) { + if (!StringUtils.hasText(profile)) { + return YamlProcessor.MatchStatus.FOUND; + } + } + return YamlProcessor.MatchStatus.NOT_FOUND; + } + + } + + /** + * Class for binding {@code spring.profiles} property. + */ + static class SpringProperties { + + private List profiles = new ArrayList(); + + public List getProfiles() { + return this.profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } + + } + +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java new file mode 100644 index 00000000..819c99fc --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java @@ -0,0 +1,419 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.io.Reader; +import java.util.*; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.util.Assert; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.parser.ParserException; +import org.yaml.snakeyaml.reader.UnicodeReader; + +/** + * Base class for YAML factories. + * + * @author Dave Syer + * @author Juergen Hoeller + * @since 4.1 + */ +public abstract class YamlProcessor { + + private final Log logger = LogFactory.getLog(getClass()); + + private ResolutionMethod resolutionMethod = ResolutionMethod.OVERRIDE; + + private Resource[] resources = new Resource[0]; + + private List documentMatchers = Collections.emptyList(); + + private boolean matchDefault = true; + + /** + * A map of document matchers allowing callers to selectively use only some of + * the documents in a YAML resource. In YAML documents are separated by + * --- lines, and each document is converted to properties before + * the match is made. E.g. + * + *

+     * environment: dev
+     * url: http://dev.bar.com
+     * name: Developer Setup
+     * ---
+     * environment: prod
+     * url:http://foo.bar.com
+     * name: My Cool App
+     * 
+ * + * when mapped with + * + *
+     * setDocumentMatchers(properties -> ("prod"
+     *     .equals(properties.getProperty("environment")) ? MatchStatus.FOUND : MatchStatus.NOT_FOUND));
+     * 
+ * + * would end up as + * + *
+     * environment=prod
+     * url=http://foo.bar.com
+     * name=My Cool App
+     * 
+ */ + public void setDocumentMatchers(DocumentMatcher... matchers) { + this.documentMatchers = Arrays.asList(matchers); + } + + /** + * Flag indicating that a document for which all the + * {@link #setDocumentMatchers(DocumentMatcher...) document matchers} abstain + * will nevertheless match. Default is {@code true}. + */ + public void setMatchDefault(boolean matchDefault) { + this.matchDefault = matchDefault; + } + + /** + * Method to use for resolving resources. Each resource will be converted to a + * Map, so this property is used to decide which map entries to keep in the + * final output from this factory. Default is {@link ResolutionMethod#OVERRIDE}. + */ + public void setResolutionMethod(ResolutionMethod resolutionMethod) { + Assert.notNull(resolutionMethod, "ResolutionMethod must not be null"); + this.resolutionMethod = resolutionMethod; + } + + /** + * Set locations of YAML {@link Resource resources} to be loaded. + * + * @see ResolutionMethod + */ + public void setResources(Resource... resources) { + this.resources = resources; + } + + /** + * Provide an opportunity for subclasses to process the Yaml parsed from the + * supplied resources. Each resource is parsed in turn and the documents inside + * checked against the {@link #setDocumentMatchers(DocumentMatcher...) + * matchers}. If a document matches it is passed into the callback, along with + * its representation as Properties. Depending on the + * {@link #setResolutionMethod(ResolutionMethod)} not all of the documents will + * be parsed. + * + * @param callback a callback to delegate to once matching documents are found + * @see #createYaml() + */ + protected void process(MatchCallback callback) { + Yaml yaml = createYaml(); + for (Resource resource : this.resources) { + boolean found = process(callback, yaml, resource); + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND && found) { + return; + } + } + } + + /** + * Create the {@link Yaml} instance to use. + */ + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor()); + } + + private boolean process(MatchCallback callback, Yaml yaml, Resource resource) { + int count = 0; + try { + if (logger.isDebugEnabled()) { + logger.debug("Loading from YAML: " + resource); + } + Reader reader = new UnicodeReader(resource.getInputStream()); + try { + for (Object object : yaml.loadAll(reader)) { + if (object != null && process(asMap(object), callback)) { + count++; + if (this.resolutionMethod == ResolutionMethod.FIRST_FOUND) { + break; + } + } + } + if (logger.isDebugEnabled()) { + logger.debug( + "Loaded " + count + " document" + (count > 1 ? "s" : "") + " from YAML resource: " + resource); + } + } finally { + reader.close(); + } + } catch (IOException ex) { + handleProcessError(resource, ex); + } + return (count > 0); + } + + private void handleProcessError(Resource resource, IOException ex) { + if (this.resolutionMethod != ResolutionMethod.FIRST_FOUND + && this.resolutionMethod != ResolutionMethod.OVERRIDE_AND_IGNORE) { + throw new IllegalStateException(ex); + } + if (logger.isWarnEnabled()) { + logger.warn("Could not load map from " + resource + ": " + ex.getMessage()); + } + } + + @SuppressWarnings("unchecked") + private Map asMap(Object object) { + // YAML can have numbers as keys + Map result = new LinkedHashMap(); + if (!(object instanceof Map)) { + // A document can be a text literal + result.put("document", object); + return result; + } + + Map map = (Map) object; + for (Map.Entry entry : map.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Map) { + value = asMap(value); + } + Object key = entry.getKey(); + if (key instanceof CharSequence) { + result.put(key.toString(), value); + } else { + // It has to be a map key in this case + result.put("[" + key.toString() + "]", value); + } + } + return result; + } + + private boolean process(Map map, MatchCallback callback) { + Properties properties = new Properties() { + + @Override + public String getProperty(String key) { + Object value = get(key); + return (value != null ? value.toString() : null); + } + }; + properties.putAll(getFlattenedMap(map)); + + if (this.documentMatchers.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Merging document (no matchers set): " + map); + } + callback.process(properties, map); + return true; + } + + MatchStatus result = MatchStatus.ABSTAIN; + for (DocumentMatcher matcher : this.documentMatchers) { + MatchStatus match = matcher.matches(properties); + result = MatchStatus.getMostSpecific(match, result); + if (match == MatchStatus.FOUND) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with document matcher: " + properties); + } + callback.process(properties, map); + return true; + } + } + + if (result == MatchStatus.ABSTAIN && this.matchDefault) { + if (logger.isDebugEnabled()) { + logger.debug("Matched document with default matcher: " + map); + } + callback.process(properties, map); + return true; + } + + if (logger.isDebugEnabled()) { + logger.debug("Unmatched document: " + map); + } + return false; + } + + /** + * Return a flattened version of the given map, recursively following any nested + * Map or Collection values. Entries from the resulting map retain the same + * order as the source. When called with the Map from a {@link MatchCallback} + * the result will contain the same values as the {@link MatchCallback} + * Properties. + * + * @param source the source map + * @return a flattened map + * @since 4.1.3 + */ + protected final Map getFlattenedMap(Map source) { + Map result = new LinkedHashMap(); + buildFlattenedMap(result, source, null); + return result; + } + + private static boolean containsText(CharSequence str) { + int strLen = str.length(); + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + return false; + } + + private void buildFlattenedMap(Map result, Map source, String path) { + for (Map.Entry entry : source.entrySet()) { + String key = entry.getKey(); + if (path != null && !path.isEmpty() && containsText(path)) { + if (key.startsWith("[")) { + key = path + key; + } else { + key = path + '.' + key; + } + } + Object value = entry.getValue(); + if (value instanceof String) { + result.put(key, value); + } else if (value instanceof Map) { + // Need a compound key + @SuppressWarnings("unchecked") + Map map = (Map) value; + buildFlattenedMap(result, map, key); + } else if (value instanceof Collection) { + // Need a compound key + @SuppressWarnings("unchecked") + Collection collection = (Collection) value; + int count = 0; + for (Object object : collection) { + buildFlattenedMap(result, Collections.singletonMap("[" + (count++) + "]", object), key); + } + } else { + result.put(key, (value != null ? value : "")); + } + } + } + + /** + * Callback interface used to process the YAML parsing results. + */ + public interface MatchCallback { + + /** + * Process the given representation of the parsing results. + * + * @param properties the properties to process (as a flattened representation + * with indexed keys in case of a collection or map) + * @param map the result map (preserving the original value structure in the + * YAML document) + */ + void process(Properties properties, Map map); + } + + /** + * Strategy interface used to test if properties match. + */ + public interface DocumentMatcher { + + /** + * Test if the given properties match. + * + * @param properties the properties to test + * @return the status of the match + */ + MatchStatus matches(Properties properties); + } + + /** + * Status returned from {@link DocumentMatcher#matches(Properties)} + */ + public enum MatchStatus { + + /** + * A match was found. + */ + FOUND, + + /** + * No match was found. + */ + NOT_FOUND, + + /** + * The matcher should not be considered. + */ + ABSTAIN; + + /** + * Compare two {@link MatchStatus} items, returning the most specific status. + */ + public static MatchStatus getMostSpecific(MatchStatus a, MatchStatus b) { + return (a.ordinal() < b.ordinal() ? a : b); + } + } + + /** + * Method to use for resolving resources. + */ + public enum ResolutionMethod { + + /** + * Replace values from earlier in the list. + */ + OVERRIDE, + + /** + * Replace values from earlier in the list, ignoring any failures. + */ + OVERRIDE_AND_IGNORE, + + /** + * Take the first resource in the list that exists and use just that. + */ + FIRST_FOUND + } + + /** + * A specialized {@link Constructor} that checks for duplicate keys. + */ + protected static class StrictMapAppenderConstructor extends Constructor { + + // Declared as public for use in subclasses + public StrictMapAppenderConstructor(){ + super(); + } + + @Override + protected Map constructMapping(MappingNode node) { + try { + return super.constructMapping(node); + } catch (IllegalStateException ex) { + throw new ParserException("while parsing MappingNode", + node.getStartMark(), + ex.getMessage(), + node.getEndMark()); + } + } + + @Override + protected Map createDefaultMap() { + final Map delegate = super.createDefaultMap(); + return new AbstractMap() { + + @Override + public Object put(Object key, Object value) { + if (delegate.containsKey(key)) { + throw new IllegalStateException("Duplicate key: " + key); + } + return delegate.put(key, value); + } + + @Override + public Set> entrySet() { + return delegate.entrySet(); + } + }; + } + } +} diff --git a/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java new file mode 100644 index 00000000..936bdba6 --- /dev/null +++ b/client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java @@ -0,0 +1,87 @@ +package com.alibaba.otter.canal.client.adapter.config.common; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Pattern; + +import org.springframework.util.ClassUtils; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.representer.Representer; +import org.yaml.snakeyaml.resolver.Resolver; + +/** + * Strategy to load '.yml' (or '.yaml') files into a {@link PropertySource}. + * + * @author Dave Syer + * @author Phillip Webb + * @author Andy Wilkinson + */ +public class YamlPropertySourceLoader implements PropertySourceLoader { + + @Override + public String[] getFileExtensions() { + return new String[] { "yml", "yaml" }; + } + + @Override + public PropertySource load(String name, Resource resource, String profile) throws IOException { + if (ClassUtils.isPresent("org.yaml.snakeyaml.Yaml", null)) { + Processor processor = new Processor(resource, profile); + Map source = processor.process(); + if (!source.isEmpty()) { + return new MapPropertySource(name, source); + } + } + return null; + } + + /** + * {@link YamlProcessor} to create a {@link Map} containing the property values. + * Similar to {@link YamlPropertiesFactoryBean} but retains the order of + * entries. + */ + private static class Processor extends YamlProcessor { + + Processor(Resource resource, String profile){ + if (profile == null) { + setMatchDefault(true); + setDocumentMatchers(new SpringProfileDocumentMatcher()); + } else { + setMatchDefault(false); + setDocumentMatchers(new SpringProfileDocumentMatcher(profile)); + } + setResources(resource); + } + + @Override + protected Yaml createYaml() { + return new Yaml(new StrictMapAppenderConstructor(), new Representer(), new DumperOptions(), new Resolver() { + + @Override + public void addImplicitResolver(Tag tag, Pattern regexp, String first) { + if (tag == Tag.TIMESTAMP) { + return; + } + super.addImplicitResolver(tag, regexp, first); + } + }); + } + + public Map process() { + final Map result = new LinkedHashMap(); + process(new MatchCallback() { + + @Override + public void process(Properties properties, Map map) { + result.putAll(getFlattenedMap(map)); + } + }); + return result; + } + + } +} diff --git a/client-adapter/elasticsearch/pom.xml b/client-adapter/elasticsearch/pom.xml index 35711014..90c64f80 100644 --- a/client-adapter/elasticsearch/pom.xml +++ b/client-adapter/elasticsearch/pom.xml @@ -18,12 +18,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.19 - provided - com.alibaba.fastsql fastsql @@ -92,4 +86,4 @@ - \ No newline at end of file + diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java index a892cf04..b22ecdca 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java @@ -1,10 +1,7 @@ package com.alibaba.otter.canal.client.adapter.es; import java.net.InetAddress; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,11 +25,7 @@ import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor; import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService; import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService; import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.SPI; +import com.alibaba.otter.canal.client.adapter.support.*; /** * ES外部适配器 @@ -69,9 +62,9 @@ public class ESAdapter implements OuterAdapter { } @Override - public void init(OuterAdapterConfig configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { try { - Map esSyncConfigTmp = ESSyncConfigLoader.load(); + Map esSyncConfigTmp = ESSyncConfigLoader.load(envProperties); // 过滤不匹配的key的配置 esSyncConfigTmp.forEach((key, config) -> { if ((config.getOuterAdapterKey() == null && configuration.getKey() == null) diff --git a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java index fcf94105..d721cc3d 100644 --- a/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java +++ b/client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java @@ -2,12 +2,12 @@ package com.alibaba.otter.canal.client.adapter.es.config; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -20,17 +20,15 @@ public class ESSyncConfigLoader { private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class); - @SuppressWarnings("unchecked") - public static synchronized Map load() { + public static synchronized Map load(Properties envProperties) { logger.info("## Start loading es mapping config ... "); Map esSyncConfig = new LinkedHashMap<>(); Map configContentMap = MappingConfigsLoader.loadConfigs("es"); configContentMap.forEach((fileName, content) -> { - Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定 - JSONObject configJson = new JSONObject(configMap); - ESSyncConfig config = configJson.toJavaObject(ESSyncConfig.class); + ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties); + try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java index 7ab197d1..a66acbb9 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java @@ -21,7 +21,7 @@ public class ConfigLoadTest { @Test public void testLoad() { - Map configMap = ESSyncConfigLoader.load(); + Map configMap = ESSyncConfigLoader.load(null); ESSyncConfig config = configMap.get("mytest_user.yml"); Assert.assertNotNull(config); Assert.assertEquals("defaultDS", config.getDataSourceKey()); diff --git a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java index 10d41de1..3f9c10d4 100644 --- a/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java +++ b/client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java @@ -26,7 +26,7 @@ public class Common { outerAdapterConfig.setProperties(properties); ESAdapter esAdapter = new ESAdapter(); - esAdapter.init(outerAdapterConfig); + esAdapter.init(outerAdapterConfig, null); return esAdapter; } diff --git a/client-adapter/hbase/pom.xml b/client-adapter/hbase/pom.xml index 29ee8b33..87e61c92 100644 --- a/client-adapter/hbase/pom.xml +++ b/client-adapter/hbase/pom.xml @@ -17,12 +17,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.19 - provided - org.apache.hbase hbase-client diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java index 95d00ca8..45b1bd51 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/HbaseAdapter.java @@ -1,10 +1,7 @@ package com.alibaba.otter.canal.client.adapter.hbase; import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import javax.sql.DataSource; @@ -58,9 +55,9 @@ public class HbaseAdapter implements OuterAdapter { } @Override - public void init(OuterAdapterConfig configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { try { - Map hbaseMappingTmp = MappingConfigLoader.load(); + Map hbaseMappingTmp = MappingConfigLoader.load(envProperties); // 过滤不匹配的key的配置 hbaseMappingTmp.forEach((key, mappingConfig) -> { if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) diff --git a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java index 3cf16a71..ff81e6b7 100644 --- a/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java +++ b/client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfigLoader.java @@ -2,12 +2,12 @@ package com.alibaba.otter.canal.client.adapter.hbase.config; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader; /** @@ -25,17 +25,15 @@ public class MappingConfigLoader { * * @return 配置名/配置文件名--对象 */ - @SuppressWarnings("unchecked") - public static Map load() { + public static Map load(Properties envProperties) { logger.info("## Start loading hbase mapping config ... "); Map result = new LinkedHashMap<>(); Map configContentMap = MappingConfigsLoader.loadConfigs("hbase"); configContentMap.forEach((fileName, content) -> { - Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定 - JSONObject configJson = new JSONObject(configMap); - MappingConfig config = configJson.toJavaObject(MappingConfig.class); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java index 80d5371a..7d082bc6 100644 --- a/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java +++ b/client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java @@ -2,10 +2,7 @@ package com.alibaba.otter.canal.adapter.launcher.loader; import java.net.InetSocketAddress; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -13,7 +10,12 @@ import java.util.concurrent.Future; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.env.Environment; +import org.springframework.core.env.PropertySource; +import org.springframework.core.env.StandardEnvironment; +import com.alibaba.otter.canal.adapter.launcher.config.SpringContext; import com.alibaba.otter.canal.client.adapter.OuterAdapter; import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig; import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader; @@ -62,7 +64,7 @@ public class CanalAdapterLoader { for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) { List canalOutConnectors = new ArrayList<>(); for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) { - loadConnector(c, canalOutConnectors); + loadAdapter(c, canalOutConnectors); } canalOuterAdapterGroups.add(canalOutConnectors); } @@ -91,7 +93,7 @@ public class CanalAdapterLoader { List> canalOuterAdapterGroups = new ArrayList<>(); List canalOuterAdapters = new ArrayList<>(); for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadConnector(config, canalOuterAdapters); + loadAdapter(config, canalOuterAdapters); } canalOuterAdapterGroups.add(canalOuterAdapters); @@ -103,8 +105,8 @@ public class CanalAdapterLoader { canalClientConfig.getFlatMessage()); canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker); canalKafkaWorker.start(); - logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-" - + group.getGroupId()); + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); } } } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) { @@ -114,7 +116,7 @@ public class CanalAdapterLoader { List> canalOuterAdapterGroups = new ArrayList<>(); List canalOuterAdapters = new ArrayList<>(); for (OuterAdapterConfig config : group.getOuterAdapters()) { - loadConnector(config, canalOuterAdapters); + loadAdapter(config, canalOuterAdapters); } canalOuterAdapterGroups.add(canalOuterAdapters); CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig, @@ -128,14 +130,14 @@ public class CanalAdapterLoader { canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker); rocketMQWorker.start(); - logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-" - + group.getGroupId()); + logger.info("Start adapter for canal-client mq topic: {} succeed", + canalAdapter.getInstance() + "-" + group.getGroupId()); } } } } - private void loadConnector(OuterAdapterConfig config, List canalOutConnectors) { + private void loadAdapter(OuterAdapterConfig config, List canalOutConnectors) { try { OuterAdapter adapter; adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey())); @@ -143,7 +145,23 @@ public class CanalAdapterLoader { ClassLoader cl = Thread.currentThread().getContextClassLoader(); // 替换ClassLoader Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader()); - adapter.init(config); + Environment env = (Environment) SpringContext.getBean(Environment.class); + Properties evnProperties = null; + if (env instanceof StandardEnvironment) { + evnProperties = new Properties(); + for (PropertySource propertySource : ((StandardEnvironment) env).getPropertySources()) { + if (propertySource instanceof EnumerablePropertySource) { + String[] names = ((EnumerablePropertySource) propertySource).getPropertyNames(); + for (String name : names) { + Object val = propertySource.getProperty(name); + if (val != null) { + evnProperties.put(name, val); + } + } + } + } + } + adapter.init(config, evnProperties); Thread.currentThread().setContextClassLoader(cl); canalOutConnectors.add(adapter); logger.info("Load canal adapter: {} succeed", config.getName()); diff --git a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java index bd7c87ab..08fd171f 100644 --- a/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java +++ b/client-adapter/logger/src/main/java/com/alibaba/otter/canal/client/adapter/logger/LoggerAdapterExample.java @@ -1,6 +1,7 @@ package com.alibaba.otter.canal.client.adapter.logger; import java.util.List; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +26,7 @@ public class LoggerAdapterExample implements OuterAdapter { private Logger logger = LoggerFactory.getLogger(this.getClass()); @Override - public void init(OuterAdapterConfig configuration) { + public void init(OuterAdapterConfig configuration, Properties envProperties) { } diff --git a/client-adapter/rdb/pom.xml b/client-adapter/rdb/pom.xml index 211950e9..0fb2c0ea 100644 --- a/client-adapter/rdb/pom.xml +++ b/client-adapter/rdb/pom.xml @@ -18,13 +18,6 @@ ${project.version} provided - - org.yaml - snakeyaml - 1.19 - provided - - mysql mysql-connector-java diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java index 2df46974..da83c6c7 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/RdbAdapter.java @@ -5,10 +5,8 @@ import java.sql.SQLException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import javax.sql.DataSource; @@ -27,12 +25,7 @@ import com.alibaba.otter.canal.client.adapter.rdb.service.RdbEtlService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbMirrorDbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.service.RdbSyncService; import com.alibaba.otter.canal.client.adapter.rdb.support.SyncUtil; -import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig; -import com.alibaba.otter.canal.client.adapter.support.Dml; -import com.alibaba.otter.canal.client.adapter.support.EtlResult; -import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig; -import com.alibaba.otter.canal.client.adapter.support.SPI; -import com.alibaba.otter.canal.client.adapter.support.Util; +import com.alibaba.otter.canal.client.adapter.support.*; /** * RDB适配器实现类 @@ -74,8 +67,8 @@ public class RdbAdapter implements OuterAdapter { * @param configuration 外部适配器配置信息 */ @Override - public void init(OuterAdapterConfig configuration) { - Map rdbMappingTmp = ConfigLoader.load(); + public void init(OuterAdapterConfig configuration, Properties envProperties) { + Map rdbMappingTmp = ConfigLoader.load(envProperties); // 过滤不匹配的key的配置 rdbMappingTmp.forEach((key, mappingConfig) -> { if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) diff --git a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java index 8d249733..63dd728a 100644 --- a/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java +++ b/client-adapter/rdb/src/main/java/com/alibaba/otter/canal/client/adapter/rdb/config/ConfigLoader.java @@ -2,8 +2,10 @@ package com.alibaba.otter.canal.client.adapter.rdb.config; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Properties; import com.alibaba.fastjson.JSONObject; +import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; @@ -25,17 +27,15 @@ public class ConfigLoader { * * @return 配置名/配置文件名--对象 */ - @SuppressWarnings("unchecked") - public static Map load() { + public static Map load(Properties envProperties) { logger.info("## Start loading rdb mapping config ... "); Map result = new LinkedHashMap<>(); Map configContentMap = MappingConfigsLoader.loadConfigs("rdb"); configContentMap.forEach((fileName, content) -> { - Map configMap = new Yaml().loadAs(content, Map.class); // yml自带的对象反射不是很稳定 - JSONObject configJson = new JSONObject(configMap); - MappingConfig config = configJson.toJavaObject(MappingConfig.class); + MappingConfig config = YmlConfigBinder + .bindYmlToObj(null, content, MappingConfig.class, null, envProperties); try { config.validate(); } catch (Exception e) { diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java index 5eeb0878..6fbf3fd3 100644 --- a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/ConfigLoadTest.java @@ -20,7 +20,7 @@ public class ConfigLoadTest { @Test public void testLoad() { - Map configMap = ConfigLoader.load(); + Map configMap = ConfigLoader.load(null); Assert.assertFalse(configMap.isEmpty()); } diff --git a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java index 9d2d117d..de949147 100644 --- a/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java +++ b/client-adapter/rdb/src/test/java/com/alibaba/otter/canal/client/adapter/rdb/test/sync/Common.java @@ -24,7 +24,7 @@ public class Common { outerAdapterConfig.setProperties(properties); RdbAdapter adapter = new RdbAdapter(); - adapter.init(outerAdapterConfig); + adapter.init(outerAdapterConfig, null); return adapter; } }