wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

= cold cleanup

Browse Source 
* using lambda if possible (we are already building in Java8)
  * using diamond when new generic instance
  * remove used improt
This commit is contained in:
zavakid
2020-09-17 17:46:20 +08:00
parent df60eda679
commit e7658879bc
133 changed files with 990 additions and 1558 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/common/Threads.java
+1 -1
View File
@@ -12,6 +12,6 @@ public class Threads {
pool,
0L,
TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(pool * 20),
new ArrayBlockingQueue<>(pool * 20),
DaemonThreadFactory.daemonThreadFactory);
}
admin/admin-web/src/main/java/com/alibaba/otter/canal/admin/model/Pager.java
+1 -1
View File
@@ -9,7 +9,7 @@ public class Pager<T> implements Serializable {
private static final long serialVersionUID = -986577815091763517L;
private Long count = 0L;
private List<T> items = new ArrayList<T>();
private List<T> items = new ArrayList<>();
private Integer page = 1;
private Integer size = 20;
private Long offset = 0L;
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/YmlConfigBinder.java
+1 -1
View File
@@ -140,7 +140,7 @@ public class YmlConfigBinder {
MutablePropertySources propertySources = new MutablePropertySources();
propertySources.addFirst(propertySource);
PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<Object>(target);
PropertiesConfigurationFactory<Object> factory = new PropertiesConfigurationFactory<>(target);
factory.setPropertySources(propertySources);
factory.setIgnoreInvalidFields(true);
factory.setIgnoreUnknownFields(true);
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/DefaultPropertyNamePatternsMatcher.java
+1 -1
View File
@@ -25,7 +25,7 @@ class DefaultPropertyNamePatternsMatcher implements PropertyNamePatternsMatcher
}
protected DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, String... names){
this(delimiters, ignoreCase, new HashSet<String>(Arrays.asList(names)));
this(delimiters, ignoreCase, new HashSet<>(Arrays.asList(names)));
}
DefaultPropertyNamePatternsMatcher(char[] delimiters, boolean ignoreCase, Set<String> names){
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertiesConfigurationFactory.java
+2 -2
View File
@@ -277,7 +277,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
}
private Set<String> getNames(Iterable<String> prefixes) {
Set<String> names = new LinkedHashSet<String>();
Set<String> names = new LinkedHashSet<>();
if (this.target != null) {
PropertyDescriptor[] descriptors = BeanUtils.getPropertyDescriptors(this.target.getClass());
for (PropertyDescriptor descriptor : descriptors) {
@@ -318,7 +318,7 @@ public class PropertiesConfigurationFactory<T> implements FactoryBean<T>, Applic
// We can filter properties to those starting with the target name, but
// we can't do a complete filter since we need to trigger the
// unknown fields check
Set<String> relaxedNames = new HashSet<String>();
Set<String> relaxedNames = new HashSet<>();
for (String relaxedTargetName : relaxedTargetNames) {
relaxedNames.add(relaxedTargetName);
}
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertyNamePatternsMatcher.java
+2 -16
View File
@@ -9,23 +9,9 @@ package com.alibaba.otter.canal.client.adapter.config.bind;
*/
interface PropertyNamePatternsMatcher {
PropertyNamePatternsMatcher ALL = new PropertyNamePatternsMatcher() {
PropertyNamePatternsMatcher ALL = propertyName -> true;
@Override
public boolean matches(String propertyName) {
return true;
}
};
PropertyNamePatternsMatcher NONE = new PropertyNamePatternsMatcher() {
@Override
public boolean matches(String propertyName) {
return false;
}
};
PropertyNamePatternsMatcher NONE = propertyName -> false;
/**
* Return {@code true} of the property name matches.
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/PropertySourcesPropertyValues.java
+2 -2
View File
@@ -37,9 +37,9 @@ public class PropertySourcesPropertyValues implements PropertyValues {
private final PropertyNamePatternsMatcher includes;
private final Map<String, PropertyValue> propertyValues = new LinkedHashMap<String, PropertyValue>();
private final Map<String, PropertyValue> propertyValues = new LinkedHashMap<>();
private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners = new ConcurrentHashMap<String, PropertySource<?>>();
private final ConcurrentHashMap<String, PropertySource<?>> collectionOwners = new ConcurrentHashMap<>();
private final boolean resolvePlaceholders;
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedDataBinder.java
+14 -14
View File
@@ -32,7 +32,7 @@ public class RelaxedDataBinder extends DataBinder {
private static final Set<Class<?>> EXCLUDED_EDITORS;
static {
Set<Class<?>> excluded = new HashSet<Class<?>>();
Set<Class<?>> excluded = new HashSet<>();
excluded.add(FileEditor.class);
EXCLUDED_EDITORS = Collections.unmodifiableSet(excluded);
}
@@ -43,7 +43,7 @@ public class RelaxedDataBinder extends DataBinder {
private boolean ignoreNestedProperties;
private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<String, String>();
private MultiValueMap<String, String> nameAliases = new LinkedMultiValueMap<>();
/**
* Create a new {@link RelaxedDataBinder} instance.
@@ -89,7 +89,7 @@ public class RelaxedDataBinder extends DataBinder {
* @param aliases a map of property name to aliases
*/
public void setNameAliases(Map<String, List<String>> aliases) {
this.nameAliases = new LinkedMultiValueMap<String, String>(aliases);
this.nameAliases = new LinkedMultiValueMap<>(aliases);
}
/**
@@ -129,8 +129,8 @@ public class RelaxedDataBinder extends DataBinder {
BeanWrapper wrapper = new BeanWrapperImpl(target);
wrapper.setConversionService(new RelaxedConversionService(getConversionService()));
wrapper.setAutoGrowNestedPaths(true);
List<PropertyValue> sortedValues = new ArrayList<PropertyValue>();
Set<String> modifiedNames = new HashSet<String>();
List<PropertyValue> sortedValues = new ArrayList<>();
Set<String> modifiedNames = new HashSet<>();
List<String> sortedNames = getSortedPropertyNames(propertyValues);
for (String name : sortedNames) {
PropertyValue propertyValue = propertyValues.getPropertyValue(name);
@@ -143,7 +143,7 @@ public class RelaxedDataBinder extends DataBinder {
}
private List<String> getSortedPropertyNames(MutablePropertyValues propertyValues) {
List<String> names = new LinkedList<String>();
List<String> names = new LinkedList<>();
for (PropertyValue propertyValue : propertyValues.getPropertyValueList()) {
names.add(propertyValue.getName());
}
@@ -161,7 +161,7 @@ public class RelaxedDataBinder extends DataBinder {
* @param names the names to sort
*/
private void sortPropertyNames(List<String> names) {
for (String name : new ArrayList<String>(names)) {
for (String name : new ArrayList<>(names)) {
int propertyIndex = names.indexOf(name);
RelaxedDataBinder.BeanPath path = new RelaxedDataBinder.BeanPath(name);
for (String prefix : path.prefixes()) {
@@ -319,7 +319,7 @@ public class RelaxedDataBinder extends DataBinder {
}
Object extend = new LinkedHashMap<String, Object>();
if (!elementDescriptor.isMap() && path.isArrayIndex(index)) {
extend = new ArrayList<Object>();
extend = new ArrayList<>();
}
wrapper.setPropertyValue(path.prefix(index + 1), extend);
}
@@ -347,7 +347,7 @@ public class RelaxedDataBinder extends DataBinder {
}
Object extend = new LinkedHashMap<String, Object>();
if (descriptor.isCollection()) {
extend = new ArrayList<Object>();
extend = new ArrayList<>();
}
if (descriptor.getType().equals(Object.class) && path.isLastNode(index)) {
extend = BLANK;
@@ -411,7 +411,7 @@ public class RelaxedDataBinder extends DataBinder {
if (aliases == null) {
return Collections.singleton(name);
}
List<String> nameAndAliases = new ArrayList<String>(aliases.size() + 1);
List<String> nameAndAliases = new ArrayList<>(aliases.size() + 1);
nameAndAliases.add(name);
nameAndAliases.addAll(aliases);
return nameAndAliases;
@@ -473,7 +473,7 @@ public class RelaxedDataBinder extends DataBinder {
}
public List<String> prefixes() {
List<String> prefixes = new ArrayList<String>();
List<String> prefixes = new ArrayList<>();
for (int index = 1; index < this.nodes.size(); index++) {
prefixes.add(prefix(index));
}
@@ -485,7 +485,7 @@ public class RelaxedDataBinder extends DataBinder {
}
private List<PathNode> splitPath(String path) {
List<PathNode> nodes = new ArrayList<PathNode>();
List<PathNode> nodes = new ArrayList<>();
String current = extractIndexedPaths(path, nodes);
for (String name : StringUtils.delimitedListToStringArray(current, ".")) {
if (StringUtils.hasText(name)) {
@@ -518,7 +518,7 @@ public class RelaxedDataBinder extends DataBinder {
}
public void collapseKeys(int index) {
List<PathNode> revised = new ArrayList<PathNode>();
List<PathNode> revised = new ArrayList<>();
for (int i = 0; i < index; i++) {
revised.add(this.nodes.get(i));
}
@@ -668,7 +668,7 @@ public class RelaxedDataBinder extends DataBinder {
private static final Set<String> BENIGN_PROPERTY_SOURCE_NAMES;
static {
Set<String> names = new HashSet<String>();
Set<String> names = new HashSet<>();
names.add(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME);
names.add(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME);
BENIGN_PROPERTY_SOURCE_NAMES = Collections.unmodifiableSet(names);
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/bind/RelaxedNames.java
+1 -1
View File
@@ -24,7 +24,7 @@ public final class RelaxedNames implements Iterable<String> {
private final String name;
private final Set<String> values = new LinkedHashSet<String>();
private final Set<String> values = new LinkedHashSet<>();
/**
* Create a new {@link RelaxedNames} instance.
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/CompositePropertySource.java
+3 -3
View File
@@ -22,7 +22,7 @@ import org.springframework.util.StringUtils;
*/
public class CompositePropertySource extends EnumerablePropertySource<Object> {
private final Set<PropertySource<?>> propertySources = new LinkedHashSet<PropertySource<?>>();
private final Set<PropertySource<?>> propertySources = new LinkedHashSet<>();
/**
* Create a new {@code CompositePropertySource}.
@@ -56,7 +56,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
@Override
public String[] getPropertyNames() {
Set<String> names = new LinkedHashSet<String>();
Set<String> names = new LinkedHashSet<>();
for (PropertySource<?> propertySource : this.propertySources) {
if (!(propertySource instanceof EnumerablePropertySource)) {
throw new IllegalStateException(
@@ -83,7 +83,7 @@ public class CompositePropertySource extends EnumerablePropertySource<Object> {
* @since 4.1
*/
public void addFirstPropertySource(PropertySource<?> propertySource) {
List<PropertySource<?>> existing = new ArrayList<PropertySource<?>>(this.propertySources);
List<PropertySource<?>> existing = new ArrayList<>(this.propertySources);
this.propertySources.clear();
this.propertySources.add(propertySource);
this.propertySources.addAll(existing);
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/MutablePropertySources.java
+1 -1
View File
@@ -28,7 +28,7 @@ public class MutablePropertySources implements PropertySources {
private final Log logger;
private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<PropertySource<?>>();
private final List<PropertySource<?>> propertySourceList = new CopyOnWriteArrayList<>();
/**
* Create a new {@link MutablePropertySources}
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/SpringProfileDocumentMatcher.java
+4 -4
View File
@@ -31,7 +31,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
}
public void addActiveProfiles(String... profiles) {
LinkedHashSet<String> set = new LinkedHashSet<String>(Arrays.asList(this.activeProfiles));
LinkedHashSet<String> set = new LinkedHashSet<>(Arrays.asList(this.activeProfiles));
Collections.addAll(set, profiles);
this.activeProfiles = set.toArray(new String[set.size()]);
}
@@ -67,14 +67,14 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
private ProfilesMatcher getProfilesMatcher() {
return (this.activeProfiles.length != 0 ? new ActiveProfilesMatcher(
new HashSet<String>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
new HashSet<>(Arrays.asList(this.activeProfiles))) : new EmptyProfilesMatcher());
}
private Set<String> extractProfiles(List<String> profiles, ProfileType type) {
if (CollectionUtils.isEmpty(profiles)) {
return null;
}
Set<String> extractedProfiles = new HashSet<String>();
Set<String> extractedProfiles = new HashSet<>();
for (String candidate : profiles) {
ProfileType candidateType = ProfileType.POSITIVE;
if (candidate.startsWith("!")) {
@@ -167,7 +167,7 @@ public class SpringProfileDocumentMatcher implements YamlProcessor.DocumentMatch
*/
static class SpringProperties {
private List<String> profiles = new ArrayList<String>();
private List<String> profiles = new ArrayList<>();
public List<String> getProfiles() {
return this.profiles;
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlProcessor.java
+2 -2
View File
@@ -166,7 +166,7 @@ public abstract class YamlProcessor {
@SuppressWarnings("unchecked")
private Map<String, Object> asMap(Object object) {
// YAML can have numbers as keys
Map<String, Object> result = new LinkedHashMap<String, Object>();
Map<String, Object> result = new LinkedHashMap<>();
if (!(object instanceof Map)) {
// A document can be a text literal
result.put("document", object);
@@ -249,7 +249,7 @@ public abstract class YamlProcessor {
* @since 4.1.3
*/
protected final Map<String, Object> getFlattenedMap(Map<String, Object> source) {
Map<String, Object> result = new LinkedHashMap<String, Object>();
Map<String, Object> result = new LinkedHashMap<>();
buildFlattenedMap(result, source, null);
return result;
}
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/config/common/YamlPropertySourceLoader.java
+2 -8
View File
@@ -72,14 +72,8 @@ public class YamlPropertySourceLoader implements PropertySourceLoader {
}
public Map<String, Object> process() {
final Map<String, Object> result = new LinkedHashMap<String, Object>();
process(new MatchCallback() {
@Override
public void process(Properties properties, Map<String, Object> map) {
result.putAll(getFlattenedMap(map));
}
});
final Map<String, Object> result = new LinkedHashMap<>();
process((properties, map) -> result.putAll(getFlattenedMap(map)));
return result;
}
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/ExtensionLoader.java
+1 -1
View File
@@ -259,7 +259,7 @@ public class ExtensionLoader<T> {
}
}
Map<String, Class<?>> extensionClasses = new HashMap<String, Class<?>>();
Map<String, Class<?>> extensionClasses = new HashMap<>();
// 1. plugin folder，customized extension classLoader （jar_dir/plugin）
String dir = File.separator + this.getJarDirectoryPath() + File.separator + "plugin";
client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java
+2 -3
View File
@@ -4,7 +4,6 @@ import java.util.*;
import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.FlatMessage;
import com.alibaba.otter.canal.protocol.Message;
/**
@@ -20,7 +19,7 @@ public class MessageUtil {
return null;
}
List<CanalEntry.Entry> entries = message.getEntries();
List<Dml> dmls = new ArrayList<Dml>(entries.size());
List<Dml> dmls = new ArrayList<>(entries.size());
for (CanalEntry.Entry entry : entries) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
@@ -136,7 +135,7 @@ public class MessageUtil {
if (commonMessages == null) {
return new ArrayList<>();
}
List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
List<Dml> dmls = new ArrayList<>(commonMessages.size());
for (CommonMessage commonMessage : commonMessages) {
Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
if (dml != null) {
client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/JdbcTypeUtilTest.java
-11
View File
@@ -1,24 +1,13 @@
package com.alibaba.otter.canal.client.adapter.support;
import static org.mockito.AdditionalMatchers.or;
import static org.mockito.Matchers.isA;
import static org.mockito.Matchers.isNull;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import com.alibaba.otter.canal.client.adapter.support.JdbcTypeUtil;
import com.alibaba.otter.canal.client.adapter.support.Util;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import java.lang.reflect.Array;
@RunWith(PowerMockRunner.class)
public class JdbcTypeUtilTest {
client-adapter/common/src/test/java/com/alibaba/otter/canal/client/adapter/support/UtilTest.java
-2
View File
@@ -3,9 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
import static org.mockito.AdditionalMatchers.or;
import static org.mockito.Matchers.isA;
import static org.mockito.Matchers.isNull;
import static org.powermock.api.mockito.PowerMockito.mockStatic;
import com.alibaba.otter.canal.client.adapter.support.Util;
import com.diffblue.deeptestutils.mock.DTUMemberMatcher;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java
+5 -10
View File
@@ -28,19 +28,14 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
private IZkChildListener childListener; // 监听所有的服务器列表
private IZkDataListener dataListener; // 监听当前的工作节点
private ZkClientx zkClient;
private volatile List<InetSocketAddress> currentAddress = new ArrayList<InetSocketAddress>();
private volatile List<InetSocketAddress> currentAddress = new ArrayList<>();
private volatile InetSocketAddress runningAddress = null;
public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){
this.destination = destination;
this.zkClient = zkClient;
childListener = new IZkChildListener() {
public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
initClusters(currentChilds);
}
};
// handleChildChange
childListener = (parentPath, currentChilds) -> initClusters(currentChilds);
dataListener = new IZkDataListener() {
@@ -80,9 +75,9 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
private void initClusters(List<String> currentChilds) {
if (currentChilds == null || currentChilds.isEmpty()) {
currentAddress = new ArrayList<InetSocketAddress>();
currentAddress = new ArrayList<>();
} else {
List<InetSocketAddress> addresses = new ArrayList<InetSocketAddress>();
List<InetSocketAddress> addresses = new ArrayList<>();
for (String address : currentChilds) {
String[] strs = StringUtils.split(address, ":");
if (strs != null && strs.length == 2) {
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java
+1 -1
View File
@@ -64,7 +64,7 @@ public class SimpleCanalConnector implements CanalConnector {
private SocketChannel channel;
private ReadableByteChannel readableChannel;
private WritableByteChannel writableChannel;
private List<Compression> supportedCompressions = new ArrayList<Compression>();
private List<Compression> supportedCompressions = new ArrayList<>();
private ClientIdentity clientIdentity;
private ClientRunningMonitor runningMonitor; // 运行控制
private ZkClientx zkClientx;
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java
+1 -1
View File
@@ -14,7 +14,7 @@ import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
*/
public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
private List<SocketAddress> nodes = new ArrayList<SocketAddress>();
private List<SocketAddress> nodes = new ArrayList<>();
private int index = 0;
public SimpleNodeAccessStrategy(List<? extends SocketAddress> nodes){
client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java
+1 -6
View File
@@ -72,12 +72,7 @@ public class ClientRunningMonitor extends AbstractCanalLifeCycle {
initRunning();
} else {
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
delayExector.schedule(new Runnable() {
public void run() {
initRunning();
}
}, delayTime, TimeUnit.SECONDS);
delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
}
}
client/src/main/java/com/alibaba/otter/canal/client/kafka/KafkaCanalConnector.java
+2 -2
View File
@@ -80,11 +80,11 @@ public class KafkaCanalConnector implements CanalMQConnector {
connected = true;
if (kafkaConsumer == null && !flatMessage) {
kafkaConsumer = new KafkaConsumer<String, Message>(properties);
kafkaConsumer = new KafkaConsumer<>(properties);
}
if (kafkaConsumer2 == null && flatMessage) {
kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
kafkaConsumer2 = new KafkaConsumer<>(properties);
}
}
client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java
+21 -33
View File
@@ -59,49 +59,37 @@ public class ClientRunningTest extends AbstractZkTest {
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
final ExecutorService executor = Executors.newFixedThreadPool(3);
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor1.stop();
sleep(2000L + RandomUtils.nextInt(500));
executor.submit(() -> {
for (int i = 0; i < 10; i++) {
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor1.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
});
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor2.stop();
sleep(2000L + RandomUtils.nextInt(500));
executor.submit(() -> {
for (int i = 0; i < 10; i++) {
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor2.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
});
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor3.stop();
sleep(2000L + RandomUtils.nextInt(500));
executor.submit(() -> {
for (int i = 0; i < 10; i++) {
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
runningMonitor3.stop();
sleep(2000L + RandomUtils.nextInt(500));
}
});
sleep(30000L);
common/src/main/java/com/alibaba/otter/canal/common/utils/ExecutorTemplate.java
+1 -1
View File
@@ -34,7 +34,7 @@ public class ExecutorTemplate {
private volatile List<Future> futures = null;
public ExecutorTemplate(ThreadPoolExecutor executor){
this.futures = Collections.synchronizedList(new ArrayList<Future>());
this.futures = Collections.synchronizedList(new ArrayList<>());
this.executor = executor;
}
common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java
+1 -7
View File
@@ -75,13 +75,7 @@ public class JsonUtils {
SerializeWriter out = new SerializeWriter();
try {
JSONSerializer serializer = new JSONSerializer(out);
serializer.getPropertyFilters().add(new PropertyFilter() {
public boolean apply(Object source, String name, Object value) {
return !propertyFliters.contains(name);
}
});
serializer.getPropertyFilters().add((source, name, value) -> !propertyFliters.contains(name));
serializer.write(obj);
return out.toString();
} finally {
common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java
+7 -11
View File
@@ -19,18 +19,14 @@ public class NamedThreadFactory implements ThreadFactory {
final private boolean daemon;
final private ThreadGroup group;
final private AtomicInteger threadNumber = new AtomicInteger(0);
final static UncaughtExceptionHandler uncaughtExceptionHandler = new UncaughtExceptionHandler() {
final static UncaughtExceptionHandler uncaughtExceptionHandler = (t, e) -> {
if (e instanceof InterruptedException
|| (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
return;
}
public void uncaughtException(Thread t,
Throwable e) {
if (e instanceof InterruptedException
|| (e.getCause() != null && e.getCause() instanceof InterruptedException)) {
return;
}
logger.error("from " + t.getName(), e);
}
};
logger.error("from " + t.getName(), e);
};
public NamedThreadFactory(){
this(DEFAULT_NAME, true);
common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java
+1 -1
View File
@@ -53,7 +53,7 @@ public final class UriUtils {
return Collections.EMPTY_MAP;
}
String query = uri.getRawQuery();
HashMap<String, String> params = new HashMap<String, String>();
HashMap<String, String> params = new HashMap<>();
@SuppressWarnings("resource")
Scanner scan = new Scanner(query);
scan.useDelimiter(SPLIT);
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
+1 -6
View File
@@ -23,12 +23,7 @@ import com.google.common.collect.MigrateMap;
public class ZkClientx extends ZkClient {
// 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>() {
public ZkClientx apply(String servers) {
return new ZkClientx(servers);
}
});
private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(ZkClientx::new);
public static ZkClientx getZkClient(String servers) {
return clients.get(servers);
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java
+1 -6
View File
@@ -73,12 +73,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
initRunning();
} else {
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
delayExector.schedule(new Runnable() {
public void run() {
initRunning();
}
}, delayTime, TimeUnit.SECONDS);
delayExector.schedule(() -> initRunning(), delayTime, TimeUnit.SECONDS);
}
}
common/src/main/java/com/google/common/collect/MigrateMap.java
+1 -1
View File
@@ -24,7 +24,7 @@ public class MigrateMap {
}
});
return new MigrateConcurrentMap<K, V>(computingCache);
return new MigrateConcurrentMap<>(computingCache);
}
public static <K, V> ConcurrentMap<K, V> makeComputingMap(Function<? super K, ? extends V> computingFunction) {
common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java
+27 -39
View File
@@ -57,55 +57,43 @@ public class ServerRunningTest extends AbstractZkTest {
final ServerRunningMonitor runningMonitor2 = buildServerRunning(countLatch, "127.0.0.1", 2089);
final ServerRunningMonitor runningMonitor3 = buildServerRunning(countLatch, "127.0.0.1", 2090);
final ExecutorService executor = Executors.newFixedThreadPool(3);
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor1.check()) {
runningMonitor1.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
executor.submit(() -> {
for (int i = 0; i < 10; i++) {
if (!runningMonitor1.isStart()) {
runningMonitor1.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor1.check()) {
runningMonitor1.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
}
});
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor2.check()) {
runningMonitor2.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
executor.submit(() -> {
for (int i = 0; i < 10; i++) {
if (!runningMonitor2.isStart()) {
runningMonitor2.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor2.check()) {
runningMonitor2.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
}
});
executor.submit(new Runnable() {
public void run() {
for (int i = 0; i < 10; i++) {
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor3.check()) {
runningMonitor3.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
executor.submit(() -> {
for (int i = 0; i < 10; i++) {
if (!runningMonitor3.isStart()) {
runningMonitor3.start();
}
sleep(2000L + RandomUtils.nextInt(500));
if (runningMonitor3.check()) {
runningMonitor3.stop();
}
sleep(2000L + RandomUtils.nextInt(500));
}
});
sleep(30000L);
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java
+3 -3
View File
@@ -42,7 +42,7 @@ public class AviaterRegexFilter {
this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null;
if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>();
list = new ArrayList<>();
} else {
String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss);
@@ -66,7 +66,7 @@ public class AviaterRegexFilter {
return defaultEmptyValue;
}
Map<String, Object> env = new HashMap<String, Object>();
Map<String, Object> env = new HashMap<>();
env.put("pattern", pattern);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
@@ -112,7 +112,7 @@ public class AviaterRegexFilter {
*/
private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>();
List<String> result = new ArrayList<>();
for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^");
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java
+2 -2
View File
@@ -35,7 +35,7 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
parallelBuildThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelBuildThreadSize * 2),
new ArrayBlockingQueue<>(parallelBuildThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Builder"),
new ThreadPoolExecutor.CallerRunsPolicy());
@@ -44,7 +44,7 @@ public abstract class AbstractMQProducer implements CanalMQProducer {
parallelSendThreadSize,
0,
TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(parallelSendThreadSize * 2),
new ArrayBlockingQueue<>(parallelSendThreadSize * 2),
new NamedThreadFactory("MQ-Parallel-Sender"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java
+62 -76
View File
@@ -75,64 +75,58 @@ public class MQMessageUtils {
private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
.softValues(),
new Function<String, List<DynamicTopicData>>() {
pkHashConfigs -> {
List<DynamicTopicData> datas = Lists.newArrayList();
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
// schema.table
for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData();
public List<DynamicTopicData> apply(String pkHashConfigs) {
List<DynamicTopicData> datas = Lists.newArrayList();
String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
",",
";"),
";");
// schema.table
for (String dynamicTopic : dynamicTopicArray) {
DynamicTopicData data = new DynamicTopicData();
if (!isWildCard(dynamicTopic)) {
data.simpleName = dynamicTopic;
} else {
if (dynamicTopic.contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
} else {
data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
}
}
datas.add(data);
}
if (!isWildCard(dynamicTopic)) {
data.simpleName = dynamicTopic;
} else {
if (dynamicTopic.contains("\\.")) {
data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
} else {
data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
}
}
datas.add(data);
}
return datas;
}
});
return datas;
});
private static Map<String, List<TopicPartitionData>> topicPartitionDatas = MigrateMap.makeComputingMap(CacheBuilder.newBuilder()
.softValues(),
new Function<String, List<TopicPartitionData>>() {
public List<TopicPartitionData> apply(String tPConfigs) {
List<TopicPartitionData> datas = Lists.newArrayList();
String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs,
",",
";"),
";");
for (String tPConfig : tPArray) {
TopicPartitionData data = new TopicPartitionData();
int i = tPConfig.lastIndexOf(":");
if (i > 0) {
String tStr = tPConfig.substring(0, i);
String pStr = tPConfig.substring(i + 1);
if (!isWildCard(tStr)) {
data.simpleName = tStr;
} else {
data.regexFilter = new AviaterRegexFilter(tStr);
}
if (!StringUtils.isEmpty(pStr) && StringUtils.isNumeric(pStr)) {
data.partitionNum = Integer.valueOf(pStr);
}
datas.add(data);
tPConfigs -> {
List<TopicPartitionData> datas = Lists.newArrayList();
String[] tPArray = StringUtils.split(StringUtils.replace(tPConfigs,
",",
";"),
";");
for (String tPConfig : tPArray) {
TopicPartitionData data = new TopicPartitionData();
int i = tPConfig.lastIndexOf(":");
if (i > 0) {
String tStr = tPConfig.substring(0, i);
String pStr = tPConfig.substring(i + 1);
if (!isWildCard(tStr)) {
data.simpleName = tStr;
} else {
data.regexFilter = new AviaterRegexFilter(tStr);
}
if (!StringUtils.isEmpty(pStr) && StringUtils.isNumeric(pStr)) {
data.partitionNum = Integer.valueOf(pStr);
}
datas.add(data);
}
return datas;
}
return datas;
});
/**
@@ -206,19 +200,15 @@ public class MQMessageUtils {
int i = 0;
for (ByteString byteString : rawEntries) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
try {
Entry entry = Entry.parseFrom(byteString);
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
datas[index] = new EntryRowData();
datas[index].entry = entry;
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
template.submit(() -> {
try {
Entry entry = Entry.parseFrom(byteString);
RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
datas[index] = new EntryRowData();
datas[index].entry = entry;
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
});
@@ -232,18 +222,14 @@ public class MQMessageUtils {
int i = 0;
for (Entry entry : message.getEntries()) {
final int index = i;
template.submit(new Runnable() {
@Override
public void run() {
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
datas[index] = new EntryRowData();
datas[index].entry = entry;
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
template.submit(() -> {
try {
RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
datas[index] = new EntryRowData();
datas[index].entry = entry;
datas[index].rowChange = rowChange;
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
});
@@ -679,7 +665,7 @@ public class MQMessageUtils {
CanalEntry.Entry entry) {
Message message = messageMap.get(topicName);
if (message == null) {
message = new Message(messageId, new ArrayList<CanalEntry.Entry>());
message = new Message(messageId, new ArrayList<>());
messageMap.put(topicName, message);
}
message.getEntries().add(entry);
connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java
+1 -7
View File
@@ -120,13 +120,7 @@ public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQ
final String topicName = entry.getKey().replace('.', '_');
final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
template.submit(new Runnable() {
@Override
public void run() {
send(destination, topicName, messageSub);
}
});
template.submit(() -> send(destination, topicName, messageSub));
}
template.waitForResult();
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonDiffConversion.java
+1 -1
View File
@@ -37,7 +37,7 @@ public class JsonDiffConversion {
public static StringBuilder print_json_diff(LogBuffer buffer, long len, String columnName, int columnIndex,
String charsetName) {
int position = buffer.position();
List<String> operation_names = new ArrayList<String>();
List<String> operation_names = new ArrayList<>();
while (buffer.hasRemaining()) {
int operation_int = buffer.getUint8();
if (operation_int >= JSON_DIFF_OPERATION_COUNT) {
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java
+1 -1
View File
@@ -16,7 +16,7 @@ import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
*/
public final class LogContext {
private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<>();
private FormatDescriptionLogEvent formatDescription;
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java
+6 -6
View File
@@ -601,7 +601,7 @@ public final class TableMapLogEvent extends LogEvent {
private void parse_signedness(LogBuffer buffer, int length) {
// stores the signedness flags extracted from field
List<Boolean> datas = new ArrayList<Boolean>();
List<Boolean> datas = new ArrayList<>();
for (int i = 0; i < length; i++) {
int ut = buffer.getUint8();
for (int c = 0x80; c != 0; c >>= 1) {
@@ -622,7 +622,7 @@ public final class TableMapLogEvent extends LogEvent {
// stores collation numbers extracted from field.
int limit = buffer.position() + length;
this.default_charset = (int) buffer.getPackedLong();
List<TableMapLogEvent.Pair> datas = new ArrayList<TableMapLogEvent.Pair>();
List<TableMapLogEvent.Pair> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) {
int col_index = (int) buffer.getPackedLong();
int col_charset = (int) buffer.getPackedLong();
@@ -639,7 +639,7 @@ public final class TableMapLogEvent extends LogEvent {
private List<Integer> parse_column_charset(LogBuffer buffer, int length) {
// stores collation numbers extracted from field.
int limit = buffer.position() + length;
List<Integer> datas = new ArrayList<Integer>();
List<Integer> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) {
int col_charset = (int) buffer.getPackedLong();
datas.add(col_charset);
@@ -664,10 +664,10 @@ public final class TableMapLogEvent extends LogEvent {
// into a string separate vector. All of them are stored
// in 'vec'.
int limit = buffer.position() + length;
List<List<String>> datas = new ArrayList<List<String>>();
List<List<String>> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) {
int count = (int) buffer.getPackedLong();
List<String> data = new ArrayList<String>(count);
List<String> data = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
int len1 = (int) buffer.getPackedLong();
data.add(buffer.getFixString(len1));
@@ -694,7 +694,7 @@ public final class TableMapLogEvent extends LogEvent {
// stores geometry column's types extracted from field.
int limit = buffer.position() + length;
List<Integer> datas = new ArrayList<Integer>();
List<Integer> datas = new ArrayList<>();
while (buffer.hasRemaining() && buffer.position() < limit) {
int col_type = (int) buffer.getPackedLong();
datas.add(col_type);
dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/event/LogHeaderTest.java
+1 -1
View File
@@ -30,7 +30,7 @@ public class LogHeaderTest {
final LogHeader objectUnderTest = new LogHeader(type);
// Assert side effects
final HashMap<String, String> hashMap = new HashMap<String, String>();
final HashMap<String, String> hashMap = new HashMap<>();
Assert.assertEquals(hashMap, objectUnderTest.gtidMap);
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+117 -132
View File
@@ -74,12 +74,7 @@ public class CanalController {
}
public CanalController(final Properties properties){
managerClients = MigrateMap.makeComputingMap(new Function<String, PlainCanalConfigClient>() {
public PlainCanalConfigClient apply(String managerAddress) {
return getManagerClient(managerAddress);
}
});
managerClients = MigrateMap.makeComputingMap(this::getManagerClient);
// 初始化全局参数设置
globalInstanceConfig = initGlobalConfig(properties);
@@ -147,85 +142,82 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(registerIp + ":" + port);
ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap((Function<String, ServerRunningMonitor>) destination -> {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
if (canalMQStarter != null) {
canalMQStarter.startDestination(destination);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
public void processActiveEnter() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
if (canalMQStarter != null) {
canalMQStarter.startDestination(destination);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
public void processActiveExit() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (canalMQStarter != null) {
canalMQStarter.stopDestination(destination);
}
embededCanalServer.stop(destination);
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
public void handleStateChanged(KeeperState state) throws Exception {
}
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
registerIp + ":" + port);
releaseCid(path);
}
} finally {
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
runningMonitor.setZkClient(zkclientx);
}
// 触发创建一下cid节点
runningMonitor.init();
return runningMonitor;
}));
// 初始化monitor机制
@@ -303,40 +295,37 @@ public class CanalController {
}
};
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
instanceConfigMonitors = MigrateMap.makeComputingMap(mode -> {
int scanInterval = Integer.valueOf(getProperty(properties,
CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
"5"));
public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer.valueOf(getProperty(properties,
CanalConstants.CANAL_AUTO_SCAN_INTERVAL,
"5"));
if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
// 设置conf目录，默认是user.dir + conf目录组成
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir)) {
rootDir = "../conf";
}
if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
monitor.setRootConf(rootDir);
} else {
// eclipse debug模式
monitor.setRootConf("src/main/resources/");
}
return monitor;
} else if (mode.isManager()) {
ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
monitor.setConfigClient(getManagerClient(managerAddress));
return monitor;
} else {
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
if (mode.isSpring()) {
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
// 设置conf目录，默认是user.dir + conf目录组成
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir)) {
rootDir = "../conf";
}
if (StringUtils.equals("otter-canal", System.getProperty("appName"))) {
monitor.setRootConf(rootDir);
} else {
// eclipse debug模式
monitor.setRootConf("src/main/resources/");
}
return monitor;
} else if (mode.isManager()) {
ManagerInstanceConfigMonitor monitor = new ManagerInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
String managerAddress = getProperty(properties, CanalConstants.CANAL_ADMIN_MANAGER);
monitor.setConfigClient(getManagerClient(managerAddress));
return monitor;
} else {
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
}
});
}
@@ -373,27 +362,23 @@ public class CanalController {
globalConfig.setSpringXml(springXml);
}
instanceGenerator = new CanalInstanceGenerator() {
public CanalInstance generate(String destination) {
InstanceConfig config = instanceConfigs.get(destination);
if (config == null) {
throw new CanalServerException("can't find destination:" + destination);
}
if (config.getMode().isManager()) {
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else if (config.getMode().isSpring()) {
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else {
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
}
instanceGenerator = destination -> {
InstanceConfig config = instanceConfigs.get(destination);
if (config == null) {
throw new CanalServerException("can't find destination:" + destination);
}
if (config.getMode().isManager()) {
PlainCanalInstanceGenerator instanceGenerator = new PlainCanalInstanceGenerator(properties);
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else if (config.getMode().isSpring()) {
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
instanceGenerator.setSpringXml(config.getSpringXml());
return instanceGenerator.generate(destination);
} else {
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
}
};
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
+1 -7
View File
@@ -121,13 +121,7 @@ public class CanalLauncher {
}
private static void setGlobalUncaughtExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
logger.error("UnCaughtException", e);
}
});
Thread.setDefaultUncaughtExceptionHandler((t, e) -> logger.error("UnCaughtException", e));
}
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java
+10 -14
View File
@@ -88,21 +88,17 @@ public class CanalStarter {
controller = new CanalController(properties);
controller.start();
logger.info("## the canal server is running now ......");
shutdownThread = new Thread() {
public void run() {
try {
logger.info("## stop the canal server");
controller.stop();
CanalLauncher.runningLatch.countDown();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal Server:", e);
} finally {
logger.info("## canal server is down.");
}
shutdownThread = new Thread(() -> {
try {
logger.info("## stop the canal server");
controller.stop();
CanalLauncher.runningLatch.countDown();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal Server:", e);
} finally {
logger.info("## canal server is down.");
}
};
});
Runtime.getRuntime().addShutdownHook(shutdownThread);
if (canalMQProducer != null) {
deployer/src/main/java/com/alibaba/otter/canal/deployer/admin/CanalAdminController.java
+1 -1
View File
@@ -106,7 +106,7 @@ public class CanalAdminController implements CanalAdmin {
public String getRunningInstances() {
try {
Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
List<String> runningInstances = new ArrayList<String>();
List<String> runningInstances = new ArrayList<>();
instances.forEach((destination, instance) -> {
if (instance.isStart()) {
runningInstances.add(destination);
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java
+8 -17
View File
@@ -32,12 +32,7 @@ public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle impleme
private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, PlainCanal> configs = MigrateMap.makeComputingMap(new Function<String, PlainCanal>() {
public PlainCanal apply(String destination) {
return new PlainCanal();
}
});
private Map<String, PlainCanal> configs = MigrateMap.makeComputingMap(destination -> new PlainCanal());
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan"));
@@ -46,19 +41,15 @@ public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle impleme
public void start() {
super.start();
executor.scheduleWithFixedDelay(new Runnable() {
public void run() {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
executor.scheduleWithFixedDelay(() -> {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java
+21 -37
View File
@@ -39,12 +39,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>() {
public InstanceConfigFiles apply(String destination) {
return new InstanceConfigFiles(destination);
}
});
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(InstanceConfigFiles::new);
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan"));
@@ -58,19 +53,15 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
super.start();
Assert.notNull(rootConf, "root conf dir is null!");
executor.scheduleWithFixedDelay(new Runnable() {
public void run() {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
executor.scheduleWithFixedDelay(() -> {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
}, 0, scanIntervalInSecond, TimeUnit.SECONDS);
}
@@ -103,29 +94,22 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
return;
}
File[] instanceDirs = rootdir.listFiles(new FileFilter() {
public boolean accept(File pathname) {
String filename = pathname.getName();
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename);
}
File[] instanceDirs = rootdir.listFiles(pathname -> {
String filename = pathname.getName();
return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename);
});
// 扫描目录的新增
Set<String> currentInstanceNames = new HashSet<String>();
Set<String> currentInstanceNames = new HashSet<>();
// 判断目录内文件的变化
for (File instanceDir : instanceDirs) {
String destination = instanceDir.getName();
currentInstanceNames.add(destination);
File[] instanceConfigs = instanceDir.listFiles(new FilenameFilter() {
public boolean accept(File dir, String name) {
// return !StringUtils.endsWithIgnoreCase(name, ".dat");
// 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload
return StringUtils.equalsIgnoreCase(name, "instance.properties");
}
File[] instanceConfigs = instanceDir.listFiles((dir, name) -> {
// return !StringUtils.endsWithIgnoreCase(name, ".dat");
// 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload
return StringUtils.equalsIgnoreCase(name, "instance.properties");
});
if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
@@ -150,7 +134,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
// 更新内容
List<FileInfo> newFileInfo = new ArrayList<FileInfo>();
List<FileInfo> newFileInfo = new ArrayList<>();
for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
}
@@ -163,7 +147,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
}
// 判断目录是否删除
Set<String> deleteInstanceNames = new HashSet<String>();
Set<String> deleteInstanceNames = new HashSet<>();
for (String destination : actions.keySet()) {
if (!currentInstanceNames.contains(destination)) {
deleteInstanceNames.add(destination);
@@ -181,7 +165,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
// 启动成功后记录配置文件信息
InstanceConfigFiles lastFile = lastFiles.get(destination);
List<FileInfo> newFileInfo = new ArrayList<FileInfo>();
List<FileInfo> newFileInfo = new ArrayList<>();
for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
}
@@ -243,10 +227,10 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private String destination; // instance
// name
private List<FileInfo> springFile = new ArrayList<FileInfo>(); // spring的instance
private List<FileInfo> springFile = new ArrayList<>(); // spring的instance
// xml
private FileInfo rootFile; // canal.properties
private List<FileInfo> instanceFiles = new ArrayList<FileInfo>(); // instance对应的配置
private List<FileInfo> instanceFiles = new ArrayList<>(); // instance对应的配置
public InstanceConfigFiles(String destination){
this.destination = destination;
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java
+5 -5
View File
@@ -64,7 +64,7 @@ public class MysqlQueryExecutor {
ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
rsHeader.fromBytes(body);
List<FieldPacket> fields = new ArrayList<FieldPacket>();
List<FieldPacket> fields = new ArrayList<>();
for (int i = 0; i < rsHeader.getColumnCount(); i++) {
FieldPacket fp = new FieldPacket();
fp.fromBytes(readNextPacket());
@@ -73,7 +73,7 @@ public class MysqlQueryExecutor {
readEofPacket();
List<RowDataPacket> rowData = new ArrayList<RowDataPacket>();
List<RowDataPacket> rowData = new ArrayList<>();
while (true) {
body = readNextPacket();
if (body[0] == -2) {
@@ -99,7 +99,7 @@ public class MysqlQueryExecutor {
cmd.setQueryString(queryString);
byte[] bodyBytes = cmd.toBytes();
PacketManager.writeBody(channel, bodyBytes);
List<ResultSetPacket> resultSets = new ArrayList<ResultSetPacket>();
List<ResultSetPacket> resultSets = new ArrayList<>();
boolean moreResult = true;
while (moreResult) {
byte[] body = readNextPacket();
@@ -112,7 +112,7 @@ public class MysqlQueryExecutor {
ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket();
rsHeader.fromBytes(body);
List<FieldPacket> fields = new ArrayList<FieldPacket>();
List<FieldPacket> fields = new ArrayList<>();
for (int i = 0; i < rsHeader.getColumnCount(); i++) {
FieldPacket fp = new FieldPacket();
fp.fromBytes(readNextPacket());
@@ -121,7 +121,7 @@ public class MysqlQueryExecutor {
moreResult = readEofPacket();
List<RowDataPacket> rowData = new ArrayList<RowDataPacket>();
List<RowDataPacket> rowData = new ArrayList<>();
while (true) {
body = readNextPacket();
if (body[0] == -2) {
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/MysqlGTIDSet.java
+2 -2
View File
@@ -84,11 +84,11 @@ public class MysqlGTIDSet implements GTIDSet {
Map<String, UUIDSet> m;
if (gtidData == null || gtidData.length() < 1) {
m = new HashMap<String, UUIDSet>();
m = new HashMap<>();
} else {
// 存在多个GTID时会有回车符
String[] uuidStrs = gtidData.replaceAll("\n", "").split(",");
m = new HashMap<String, UUIDSet>(uuidStrs.length);
m = new HashMap<>(uuidStrs.length);
for (int i = 0; i < uuidStrs.length; i++) {
UUIDSet uuidSet = UUIDSet.parse(uuidStrs[i]);
m.put(uuidSet.SID.toString(), uuidSet);
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/UUIDSet.java
+2 -2
View File
@@ -105,7 +105,7 @@ public class UUIDSet {
throw new RuntimeException(String.format("parseUUIDSet failed due to wrong format: %s", str));
}
List<Interval> intervals = new ArrayList<Interval>();
List<Interval> intervals = new ArrayList<>();
for (int i = 1; i < ss.length; i++) {
intervals.add(parseInterval(ss[i]));
}
@@ -172,7 +172,7 @@ public class UUIDSet {
* @return
*/
public static List<Interval> combine(List<Interval> intervals) {
List<Interval> combined = new ArrayList<Interval>();
List<Interval> combined = new ArrayList<>();
Collections.sort(intervals);
int len = intervals.size();
for (int i = 0; i < len; i++) {
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java
+2 -2
View File
@@ -7,8 +7,8 @@ import java.util.List;
public class ResultSetPacket {
private SocketAddress sourceAddress;
private List<FieldPacket> fieldDescriptors = new ArrayList<FieldPacket>();
private List<String> fieldValues = new ArrayList<String>();
private List<FieldPacket> fieldDescriptors = new ArrayList<>();
private List<String> fieldValues = new ArrayList<>();
public void setFieldDescriptors(List<FieldPacket> fieldDescriptors) {
this.fieldDescriptors = fieldDescriptors;
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java
+1 -1
View File
@@ -9,7 +9,7 @@ import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader;
public class RowDataPacket extends PacketWithHeaderPacket {
private List<String> columns = new ArrayList<String>();
private List<String> columns = new ArrayList<>();
public void fromBytes(byte[] data) throws IOException {
int index = 0;
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/NettySocketChannelPool.java
+1 -1
View File
@@ -31,7 +31,7 @@ public abstract class NettySocketChannelPool {
private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组
private static Bootstrap boot = new Bootstrap(); // 主
private static Map<Channel, SocketChannel> chManager = new ConcurrentHashMap<Channel, SocketChannel>();
private static Map<Channel, SocketChannel> chManager = new ConcurrentHashMap<>();
private static final Logger logger = LoggerFactory.getLogger(NettySocketChannelPool.class);
static {
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/CharsetUtil.java
+1 -1
View File
@@ -14,7 +14,7 @@ import org.apache.commons.lang.StringUtils;
public class CharsetUtil {
private static final String[] INDEX_TO_CHARSET = new String[2048];
private static final Map<String, Integer> CHARSET_TO_INDEX = new HashMap<String, Integer>();
private static final Map<String, Integer> CHARSET_TO_INDEX = new HashMap<>();
static {
INDEX_TO_CHARSET[1] = "big5";
INDEX_TO_CHARSET[84] = "big5";
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlGTIDSetTest.java
+3 -3
View File
@@ -38,7 +38,7 @@ public class MysqlGTIDSetTest {
@Test
public void testParse() {
Map<String, MysqlGTIDSet> cases = new HashMap<String, MysqlGTIDSet>(5);
Map<String, MysqlGTIDSet> cases = new HashMap<>(5);
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1",
buildForTest(new Material("726757ad-4455-11e8-ae04-0242ac110002", 1, 2)));
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3",
@@ -89,12 +89,12 @@ public class MysqlGTIDSetTest {
}
private MysqlGTIDSet buildForTest(List<Material> materials) {
Map<String, UUIDSet> sets = new HashMap<String, UUIDSet>();
Map<String, UUIDSet> sets = new HashMap<>();
for (Material a : materials) {
UUIDSet.Interval interval = new UUIDSet.Interval();
interval.start = a.start;
interval.stop = a.stop;
List<UUIDSet.Interval> intervals = new ArrayList<UUIDSet.Interval>();
List<UUIDSet.Interval> intervals = new ArrayList<>();
intervals.add(interval);
if (a.start1 > 0 && a.stop1 > 0) {
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/UUIDSetTest.java
+1 -1
View File
@@ -16,7 +16,7 @@ public class UUIDSetTest {
@Test
public void testToString() {
Map<String, String> cases = new HashMap<String, String>(4);
Map<String, String> cases = new HashMap<>(4);
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1", "726757ad-4455-11e8-ae04-0242ac110002:1");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3", "726757ad-4455-11e8-ae04-0242ac110002:1-3");
cases.put("726757ad-4455-11e8-ae04-0242ac110002:1-3:4-6", "726757ad-4455-11e8-ae04-0242ac110002:1-6");
example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
+1 -6
View File
@@ -25,12 +25,7 @@ public class AbstractCanalClientTest extends BaseCanalClientTest {
protected void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
running = true;
example/src/main/java/com/alibaba/otter/canal/example/BaseCanalClientTest.java
+1 -6
View File
@@ -31,12 +31,7 @@ public class BaseCanalClientTest {
protected static final String SEP = SystemUtils.LINE_SEPARATOR;
protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
protected volatile boolean running = false;
protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
protected Thread thread = null;
protected CanalConnector connector;
protected static String context_format = null;
example/src/main/java/com/alibaba/otter/canal/example/ClusterCanalClientTest.java
+9 -13
View File
@@ -32,19 +32,15 @@ public class ClusterCanalClientTest extends AbstractCanalClientTest {
clientTest.setConnector(connector);
clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
});
}));
}
}
example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientPermanceTest.java
+7 -11
View File
@@ -20,23 +20,19 @@ public class SimpleCanalClientPermanceTest {
int perSum = 0;
long start = System.currentTimeMillis();
long end = 0;
final ArrayBlockingQueue<Long> queue = new ArrayBlockingQueue<Long>(100);
final ArrayBlockingQueue<Long> queue = new ArrayBlockingQueue<>(100);
try {
final CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(ip, 11111),
destination,
"canal",
"canal");
Thread ackThread = new Thread(new Runnable() {
@Override
public void run() {
while (true) {
try {
long batchId = queue.take();
connector.ack(batchId);
} catch (InterruptedException e) {
}
Thread ackThread = new Thread(() -> {
while (true) {
try {
long batchId = queue.take();
connector.ack(batchId);
} catch (InterruptedException e) {
}
}
});
example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java
+9 -13
View File
@@ -30,20 +30,16 @@ public class SimpleCanalClientTest extends AbstractCanalClientTest {
final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
clientTest.setConnector(connector);
clientTest.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## stop the canal client");
clientTest.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping canal:", e);
} finally {
logger.info("## canal client is down.");
}
});
}));
}
}
example/src/main/java/com/alibaba/otter/canal/example/kafka/CanalKafkaClientExample.java
+11 -25
View File
@@ -26,12 +26,7 @@ public class CanalKafkaClientExample {
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
@@ -47,20 +42,16 @@ public class CanalKafkaClientExample {
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
});
}));
while (running)
;
} catch (Throwable e) {
@@ -71,12 +62,7 @@ public class CanalKafkaClientExample {
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
example/src/main/java/com/alibaba/otter/canal/example/kafka/CanalKafkaClientFlatMessageExample.java
+11 -25
View File
@@ -26,12 +26,7 @@ public class CanalKafkaClientFlatMessageExample {
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public CanalKafkaClientFlatMessageExample(String zkServers, String servers, String topic, Integer partition,
String groupId){
@@ -48,20 +43,16 @@ public class CanalKafkaClientFlatMessageExample {
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
});
}));
while (running)
;
} catch (Throwable e) {
@@ -72,12 +63,7 @@ public class CanalKafkaClientFlatMessageExample {
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
example/src/main/java/com/alibaba/otter/canal/example/kafka/CanalKafkaOffsetClientExample.java
+11 -25
View File
@@ -35,12 +35,7 @@ public class CanalKafkaOffsetClientExample {
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public CanalKafkaOffsetClientExample(String servers, String topic, Integer partition, String groupId){
connector = new KafkaOffsetCanalConnector(servers, topic, partition, groupId, false);
@@ -55,20 +50,16 @@ public class CanalKafkaOffsetClientExample {
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
kafkaCanalClientExample.start();
logger.info("## the canal kafka consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## stop the kafka consumer");
kafkaCanalClientExample.stop();
} catch (Throwable e) {
logger.warn("##something goes wrong when stopping kafka consumer:", e);
} finally {
logger.info("## kafka consumer is down.");
}
});
}));
while (running)
;
} catch (Throwable e) {
@@ -79,12 +70,7 @@ public class CanalKafkaOffsetClientExample {
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
example/src/main/java/com/alibaba/otter/canal/example/kafka/KafkaClientRunningTest.java
+10 -14
View File
@@ -26,22 +26,18 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
public void testKafkaConsumer() {
final ExecutorService executor = Executors.newFixedThreadPool(1);
final KafkaCanalConnector connector = new KafkaCanalConnector(servers, topic, partition, groupId, null, false);
executor.submit(new Runnable() {
@Override
public void run() {
connector.connect();
connector.subscribe();
while (running) {
List<Message> messages = connector.getList(3L, TimeUnit.SECONDS);
if (messages != null) {
System.out.println(messages);
}
connector.ack();
executor.submit(() -> {
connector.connect();
connector.subscribe();
while (running) {
List<Message> messages = connector.getList(3L, TimeUnit.SECONDS);
if (messages != null) {
System.out.println(messages);
}
connector.unsubscribe();
connector.disconnect();
connector.ack();
}
connector.unsubscribe();
connector.disconnect();
});
sleep(60000);
example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientExample.java
+11 -25
View File
@@ -24,12 +24,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public CanalRocketMQClientExample(String nameServers, String topic, String groupId) {
connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, false);
@@ -55,20 +50,16 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
logger.info("## Start the rocketmq consumer: {}-{}", topic, groupId);
rocketMQClientExample.start();
logger.info("## The canal rocketmq consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## Stop the rocketmq consumer");
rocketMQClientExample.stop();
} catch (Throwable e) {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} finally {
logger.info("## Rocketmq consumer is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## Stop the rocketmq consumer");
rocketMQClientExample.stop();
} catch (Throwable e) {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} finally {
logger.info("## Rocketmq consumer is down.");
}
});
}));
while (running)
;
} catch (Throwable e) {
@@ -79,12 +70,7 @@ public class CanalRocketMQClientExample extends AbstractRocektMQTest {
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
example/src/main/java/com/alibaba/otter/canal/example/rocketmq/CanalRocketMQClientFlatMessageExample.java
+11 -25
View File
@@ -26,12 +26,7 @@ public class CanalRocketMQClientFlatMessageExample extends AbstractRocektMQTest
private Thread thread = null;
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
logger.error("parse events has an error", e);
}
};
private Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
public CanalRocketMQClientFlatMessageExample(String nameServers, String topic, String groupId){
connector = new RocketMQCanalConnector(nameServers, topic, groupId, 500, true);
@@ -45,20 +40,16 @@ public class CanalRocketMQClientFlatMessageExample extends AbstractRocektMQTest
logger.info("## Start the rocketmq consumer: {}-{}", topic, groupId);
rocketMQClientExample.start();
logger.info("## The canal rocketmq consumer is running now ......");
Runtime.getRuntime().addShutdownHook(new Thread() {
public void run() {
try {
logger.info("## Stop the rocketmq consumer");
rocketMQClientExample.stop();
} catch (Throwable e) {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} finally {
logger.info("## Rocketmq consumer is down.");
}
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
logger.info("## Stop the rocketmq consumer");
rocketMQClientExample.stop();
} catch (Throwable e) {
logger.warn("## Something goes wrong when stopping rocketmq consumer:", e);
} finally {
logger.info("## Rocketmq consumer is down.");
}
});
}));
while (running)
;
} catch (Throwable e) {
@@ -69,12 +60,7 @@ public class CanalRocketMQClientFlatMessageExample extends AbstractRocektMQTest
public void start() {
Assert.notNull(connector, "connector is null");
thread = new Thread(new Runnable() {
public void run() {
process();
}
});
thread = new Thread(this::process);
thread.setUncaughtExceptionHandler(handler);
thread.start();
running = true;
filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
+11 -14
View File
@@ -14,20 +14,17 @@ import com.google.common.collect.MigrateMap;
public class PatternUtils {
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(CacheBuilder.newBuilder().softValues(),
new Function<String, Pattern>() {
public Pattern apply(String pattern) {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK
| Perl5Compiler.READ_ONLY_MASK
| Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e) {
throw new RuntimeException(e);
}
}
});
pattern -> {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK
| Perl5Compiler.READ_ONLY_MASK
| Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e) {
throw new RuntimeException(e);
}
});
public static Pattern getPattern(String pattern) {
return patterns.get(pattern);
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java
+1 -1
View File
@@ -29,7 +29,7 @@ public class AviaterELFilter implements CanalEventFilter<CanalEntry.Entry> {
return true;
}
Map<String, Object> env = new HashMap<String, Object>();
Map<String, Object> env = new HashMap<>();
env.put(ROOT_KEY, entry);
return (Boolean) AviatorEvaluator.execute(expression, env);
}
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
+3 -3
View File
@@ -44,7 +44,7 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
this.defaultEmptyValue = defaultEmptyValue;
List<String> list = null;
if (StringUtils.isEmpty(pattern)) {
list = new ArrayList<String>();
list = new ArrayList<>();
} else {
String[] ss = StringUtils.split(pattern, SPLIT);
list = Arrays.asList(ss);
@@ -68,7 +68,7 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
return defaultEmptyValue;
}
Map<String, Object> env = new HashMap<String, Object>();
Map<String, Object> env = new HashMap<>();
env.put("pattern", pattern);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
@@ -114,7 +114,7 @@ public class AviaterRegexFilter implements CanalEventFilter<String> {
*/
private List<String> completionPattern(List<String> patterns) {
List<String> result = new ArrayList<String>();
List<String> result = new ArrayList<>();
for (String pattern : patterns) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("^");
filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java
+2 -2
View File
@@ -30,7 +30,7 @@ public class AviaterSimpleFilter implements CanalEventFilter<String> {
public AviaterSimpleFilter(String filterExpression){
if (StringUtils.isEmpty(filterExpression)) {
list = new ArrayList<String>();
list = new ArrayList<>();
} else {
String[] ss = filterExpression.toLowerCase().split(SPLIT);
list = Arrays.asList(ss);
@@ -44,7 +44,7 @@ public class AviaterSimpleFilter implements CanalEventFilter<String> {
if (StringUtils.isEmpty(filtered)) {
return true;
}
Map<String, Object> env = new HashMap<String, Object>();
Map<String, Object> env = new HashMap<>();
env.put("list", list);
env.put("target", filtered.toLowerCase());
return (Boolean) exp.execute(env);
filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java
+12 -15
View File
@@ -21,22 +21,19 @@ public class MutliAviaterFilterTest {
final CountDownLatch countDown = new CountDownLatch(count);
final AtomicInteger successed = new AtomicInteger(0);
for (int i = 0; i < count; i++) {
executor.submit(new Runnable() {
public void run() {
try {
for (int i = 0; i < 100; i++) {
doRegexTest();
// try {
// Thread.sleep(10);
// } catch (InterruptedException e) {
// }
}
successed.incrementAndGet();
} finally {
countDown.countDown();
executor.submit(() -> {
try {
for (int i1 = 0; i1 < 100; i1++) {
doRegexTest();
// try {
// Thread.sleep(10);
// } catch (InterruptedException e) {
// }
}
successed.incrementAndGet();
} finally {
countDown.countDown();
}
});
}
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
+5 -11
View File
@@ -118,13 +118,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
} else {
try {
File externalLibDir = new File(alarmHandlerPluginDir);
File[] jarFiles = externalLibDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".jar");
}
});
File[] jarFiles = externalLibDir.listFiles((dir, name) -> name.endsWith(".jar"));
if (jarFiles == null || jarFiles.length == 0) {
throw new IllegalStateException(String.format("alarmHandlerPluginDir [%s] can't find any name endswith \".jar\" file.",
alarmHandlerPluginDir));
@@ -242,9 +236,9 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
if (!CollectionUtils.isEmpty(groupDbAddresses)) {
int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
List<CanalEventParser> eventParsers = new ArrayList<>();
for (int i = 0; i < size; i++) {
List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>();
List<InetSocketAddress> dbAddress = new ArrayList<>();
SourcingType lastType = null;
for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
@@ -270,7 +264,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
}
} else {
// 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>());
this.eventParser = doInitEventParser(type, new ArrayList<>());
}
logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
@@ -494,7 +488,7 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
private synchronized ZkClientx getZkclientx() {
// 做一下排序，保证相同的机器只使用同一个链接
List<String> zkClusters = new ArrayList<String>(parameters.getZkClusters());
List<String> zkClusters = new ArrayList<>(parameters.getZkClusters());
Collections.sort(zkClusters);
return ZkClientx.getZkClient(StringUtils.join(zkClusters, ";"));
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
+6 -6
View File
@@ -661,7 +661,7 @@ public class CanalParameter implements Serializable {
public List<InetSocketAddress> getDbAddresses() {
if (dbAddresses == null) {
dbAddresses = new ArrayList<InetSocketAddress>();
dbAddresses = new ArrayList<>();
if (masterAddress != null) {
dbAddresses.add(masterAddress);
}
@@ -675,22 +675,22 @@ public class CanalParameter implements Serializable {
public List<List<DataSourcing>> getGroupDbAddresses() {
if (groupDbAddresses == null) {
groupDbAddresses = new ArrayList<List<DataSourcing>>();
groupDbAddresses = new ArrayList<>();
if (dbAddresses != null) {
for (InetSocketAddress address : dbAddresses) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>();
List<DataSourcing> groupAddresses = new ArrayList<>();
groupAddresses.add(new DataSourcing(sourcingType, address));
groupDbAddresses.add(groupAddresses);
}
} else {
if (masterAddress != null) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>();
List<DataSourcing> groupAddresses = new ArrayList<>();
groupAddresses.add(new DataSourcing(sourcingType, masterAddress));
groupDbAddresses.add(groupAddresses);
}
if (standbyAddress != null) {
List<DataSourcing> groupAddresses = new ArrayList<DataSourcing>();
List<DataSourcing> groupAddresses = new ArrayList<>();
groupAddresses.add(new DataSourcing(sourcingType, standbyAddress));
groupDbAddresses.add(groupAddresses);
}
@@ -731,7 +731,7 @@ public class CanalParameter implements Serializable {
public List<String> getPositions() {
if (positions == null) {
positions = new ArrayList<String>();
positions = new ArrayList<>();
String masterPosition = buildPosition(masterLogfileName, masterLogfileOffest, masterTimestamp);
if (masterPosition != null) {
positions.add(masterPosition);
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/plain/HttpHelper.java
+1 -7
View File
@@ -57,13 +57,7 @@ public class HttpHelper {
// 创建支持忽略证书的https
try {
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (x509Certificates, s) -> true).build();
httpclient = HttpClientBuilder.create()
.setSSLContext(sslContext)
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java
+2 -9
View File
@@ -23,14 +23,7 @@ public class PropertyPlaceholderConfigurer extends org.springframework.beans.fac
private static final String PLACEHOLDER_PREFIX = "${";
private static final String PLACEHOLDER_SUFFIX = "}";
public static ThreadLocal<Properties> propertiesLocal = new ThreadLocal<Properties>() {
@Override
protected Properties initialValue() {
return new Properties();
}
};
public static ThreadLocal<Properties> propertiesLocal = ThreadLocal.withInitial(Properties::new);
private ResourceLoader loader;
private String[] locationNames;
@@ -57,7 +50,7 @@ public class PropertyPlaceholderConfigurer extends org.springframework.beans.fac
}
if (locationNames != null) {
List<Resource> resources = new ArrayList<Resource>(locationNames.length);
List<Resource> resources = new ArrayList<>(locationNames.length);
for (String location : locationNames) {
location = trimToNull(location);
meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
+30 -56
View File
@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.meta;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -43,7 +44,7 @@ import com.google.common.collect.MigrateMap;
public class FileMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
private static final Logger logger = LoggerFactory.getLogger(FileMixedMetaManager.class);
private static final Charset charset = Charset.forName("UTF-8");
private static final Charset charset = StandardCharsets.UTF_8;
private File dataDir;
private String dataFileName = "meta.dat";
private Map<String, File> dataFileCaches;
@@ -69,58 +70,41 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination) {
return getDataFile(destination);
}
});
dataFileCaches = MigrateMap.makeComputingMap(this::getDataFile);
executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(this::loadClientIdentity);
public List<ClientIdentity> apply(String destination) {
return loadClientIdentity(destination);
cursors = MigrateMap.makeComputingMap(clientIdentity -> {
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity) {
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
}
});
updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
updateCursorTasks = Collections.synchronizedSet(new HashSet<>());
// 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() {
public void run() {
List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
for (ClientIdentity clientIdentity : tasks) {
MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
try {
// 定时将内存中的最新值刷到file中，多次变更只刷一次
if (logger.isInfoEnabled()) {
LogPosition cursor = (LogPosition) getCursor(clientIdentity);
logger.info("clientId:{} cursor:[{},{},{},{},{}] address[{}]", new Object[] {
clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getPostion().getServerId(), cursor.getPostion().getGtid(),
cursor.getIdentity().getSourceAddress().toString() });
}
flushDataToFile(clientIdentity.getDestination());
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
executor.scheduleAtFixedRate(() -> {
List<ClientIdentity> tasks = new ArrayList<>(updateCursorTasks);
for (ClientIdentity clientIdentity : tasks) {
MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
try {
// 定时将内存中的最新值刷到file中，多次变更只刷一次
if (logger.isInfoEnabled()) {
LogPosition cursor = (LogPosition) getCursor(clientIdentity);
logger.info("clientId:{} cursor:[{},{},{},{},{}] address[{}]", clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
cursor.getPostion().getServerId(), cursor.getPostion().getGtid(),
cursor.getIdentity().getSourceAddress().toString());
}
flushDataToFile(clientIdentity.getDestination());
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
}
}
},
@@ -142,24 +126,14 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
super.subscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
flushDataToFile(clientIdentity.getDestination());
}
});
executor.submit(() -> flushDataToFile(clientIdentity.getDestination()));
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
flushDataToFile(clientIdentity.getDestination());
}
});
executor.submit(() -> flushDataToFile(clientIdentity.getDestination()));
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java
+2 -13
View File
@@ -32,22 +32,11 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
public void start() {
super.start();
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
return MemoryClientIdentityBatch.create(clientIdentity);
}
});
batches = MigrateMap.makeComputingMap(MemoryClientIdentityBatch::create);
cursors = new MapMaker().makeMap();
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
public List<ClientIdentity> apply(String destination) {
return Lists.newArrayList();
}
});
destinations = MigrateMap.makeComputingMap(destination -> Lists.newArrayList());
}
public void stop() {
meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java
+22 -68
View File
@@ -1,6 +1,5 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -11,7 +10,6 @@ import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
/**
@@ -37,36 +35,25 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
}
executor = Executors.newFixedThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(destination -> zooKeeperMetaManager.listAllSubscribeInfo(destination));
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
cursors = MigrateMap.makeComputingMap(clientIdentity -> {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
}
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
batches = MigrateMap.makeComputingMap(clientIdentity -> {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
});
}
@@ -85,23 +72,13 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.subscribe(clientIdentity);
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
executor.submit(() -> zooKeeperMetaManager.subscribe(clientIdentity));
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
executor.submit(() -> zooKeeperMetaManager.unsubscribe(clientIdentity));
}
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
@@ -109,12 +86,7 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
super.updateCursor(clientIdentity, position);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.updateCursor(clientIdentity, position);
}
});
executor.submit(() -> zooKeeperMetaManager.updateCursor(clientIdentity, position));
}
@Override
@@ -131,12 +103,7 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
throws CanalMetaManagerException {
final Long batchId = super.addBatch(clientIdentity, positionRange);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
executor.submit(() -> zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId));
return batchId;
}
@@ -144,23 +111,15 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
throws CanalMetaManagerException {
super.addBatch(clientIdentity, positionRange, batchId);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
executor.submit(() -> zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId));
}
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
throws CanalMetaManagerException {
PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
}
executor.submit(() -> {
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
});
return positionRange;
@@ -170,12 +129,7 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
super.clearAllBatchs(clientIdentity);
// 异步刷新
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
}
});
executor.submit(() -> zooKeeperMetaManager.clearAllBatchs(clientIdentity));
}
// =============== setter / getter ================
meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java
+27 -52
View File
@@ -18,7 +18,6 @@ import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
/**
@@ -52,54 +51,40 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
}
executor = Executors.newScheduledThreadPool(1);
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(destination -> zooKeeperMetaManager.listAllSubscribeInfo(destination));
public List<ClientIdentity> apply(String destination) {
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
cursors = MigrateMap.makeComputingMap(clientIdentity -> {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
});
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {
public Position apply(ClientIdentity clientIdentity) {
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
return position;
}
batches = MigrateMap.makeComputingMap(clientIdentity -> {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
});
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
}
});
updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
updateCursorTasks = Collections.synchronizedSet(new HashSet<>());
// 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() {
public void run() {
List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
for (ClientIdentity clientIdentity : tasks) {
try {
// 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
}
executor.scheduleAtFixedRate(() -> {
List<ClientIdentity> tasks = new ArrayList<>(updateCursorTasks);
for (ClientIdentity clientIdentity : tasks) {
try {
// 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
updateCursorTasks.remove(clientIdentity);
} catch (Throwable e) {
// ignore
logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
}
}
}, period, period, TimeUnit.MILLISECONDS);
@@ -121,24 +106,14 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
super.subscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
executor.submit(() -> zooKeeperMetaManager.subscribe(clientIdentity));
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
super.unsubscribe(clientIdentity);
// 订阅信息频率发生比较低，不需要做定时merge处理
executor.submit(new Runnable() {
public void run() {
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
executor.submit(() -> zooKeeperMetaManager.unsubscribe(clientIdentity));
}
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
+6 -6
View File
@@ -103,7 +103,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
if (zkClientx == null) { //重新加载时可能为空
return new ArrayList<ClientIdentity>();
return new ArrayList<>();
}
String path = ZookeeperPathUtils.getDestinationPath(destination);
List<String> childs = null;
@@ -114,9 +114,9 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
}
if (CollectionUtils.isEmpty(childs)) {
return new ArrayList<ClientIdentity>();
return new ArrayList<>();
}
List<Short> clientIds = new ArrayList<Short>();
List<Short> clientIds = new ArrayList<>();
for (String child : childs) {
if (StringUtils.isNumeric(child)) {
clientIds.add(ZookeeperPathUtils.getClientId(child));
@@ -252,7 +252,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
return null;
}
// 找到最大的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
ArrayList<Long> batchIds = new ArrayList<>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
@@ -279,7 +279,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
return null;
}
// 找到最小的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
ArrayList<Long> batchIds = new ArrayList<>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
@@ -306,7 +306,7 @@ public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements Cana
return Maps.newHashMap();
}
// 找到最大的Id
ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
ArrayList<Long> batchIds = new ArrayList<>(nodes.size());
for (String batchIdString : nodes) {
batchIds.add(Long.valueOf(batchIdString));
}
meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java
+2 -2
View File
@@ -20,7 +20,7 @@ import org.junit.Test;
public class AbstractMetaManagerTest extends AbstractZkTest {
private static final String MYSQL_ADDRESS = "127.0.0.1";
protected ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1); ;
protected ClientIdentity clientIdentity = new ClientIdentity(destination, (short) 1);
@Test
public void doSubscribeTest(CanalMetaManager metaManager) {
@@ -116,6 +116,6 @@ public class AbstractMetaManagerTest extends AbstractZkTest {
LogPosition end = new LogPosition();
end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
return new PositionRange<LogPosition>(start, end);
return new PositionRange<>(start, end);
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
+14 -23
View File
@@ -80,14 +80,8 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
protected Thread parseThread = null;
protected Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t,
Throwable e) {
logger.error("parse events has an error",
e);
}
};
protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error",
e);
protected EventTransactionBuffer transactionBuffer;
protected int transactionSize = 1024;
@@ -134,22 +128,19 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
public AbstractEventParser(){
// 初始化一下
transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {
transactionBuffer = new EventTransactionBuffer(transaction -> {
boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
if (!running) {
return;
}
public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
if (!running) {
return;
}
if (!successed) {
throw new CanalParseException("consume failed!");
}
if (!successed) {
throw new CanalParseException("consume failed!");
}
LogPosition position = buildLastTransactionPosition(transaction);
if (position != null) { // 可能position为空
logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
}
LogPosition position = buildLastTransactionPosition(transaction);
if (position != null) { // 可能position为空
logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
}
});
}
@@ -542,7 +533,7 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
*/
private Map<String, List<String>> parseFieldFilterMap(String config) {
Map<String, List<String>> map = new HashMap<String, List<String>>();
Map<String, List<String>> map = new HashMap<>();
if (StringUtils.isNotBlank(config)) {
for (String filter : config.split(",")) {
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java
+1 -1
View File
@@ -115,7 +115,7 @@ public class EventTransactionBuffer extends AbstractCanalLifeCycle {
long end = this.putSequence.get();
if (start <= end) {
List<CanalEntry.Entry> transaction = new ArrayList<CanalEntry.Entry>();
List<CanalEntry.Entry> transaction = new ArrayList<>();
for (long next = start; next <= end; next++) {
transaction.add(this.entries[getIndex(next)]);
}
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java
+2 -2
View File
@@ -23,7 +23,7 @@ public class TableMeta {
private String schema;
private String table;
private List<FieldMeta> fields = new ArrayList<TableMeta.FieldMeta>();
private List<FieldMeta> fields = new ArrayList<>();
private String ddl; // 表结构的DDL语句
public TableMeta(){
@@ -75,7 +75,7 @@ public class TableMeta {
}
public List<FieldMeta> getPrimaryFields() {
List<FieldMeta> primarys = new ArrayList<TableMeta.FieldMeta>();
List<FieldMeta> primarys = new ArrayList<>();
for (FieldMeta meta : fields) {
if (meta.isKey()) {
primarys.add(meta);
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java
+1 -1
View File
@@ -14,7 +14,7 @@ import com.alibaba.otter.canal.parse.CanalEventParser;
*/
public class GroupEventParser extends AbstractCanalLifeCycle implements CanalEventParser {
private List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
private List<CanalEventParser> eventParsers = new ArrayList<>();
public void start() {
super.start();
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
+1 -1
View File
@@ -714,7 +714,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
}
int i = 0;
Map<String, String> maps = new HashMap<String, String>(names.size(), 1f);
Map<String, String> maps = new HashMap<>(names.size(), 1f);
for (FieldPacket name : names) {
maps.put(name.getName(), fields.get(i));
i++;
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlMultiStageCoprocessor.java
+6 -6
View File
@@ -98,9 +98,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
// stage 2
this.logContext = new LogContext();
simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
sequenceBarrier,
new SimpleParserStage(logContext));
simpleParserStage = new BatchEventProcessor<>(disruptorMsgBuffer,
sequenceBarrier,
new SimpleParserStage(logContext));
simpleParserStage.setExceptionHandler(exceptionHandler);
disruptorMsgBuffer.addGatingSequences(simpleParserStage.getSequence());
@@ -119,9 +119,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
// stage 4
SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence);
sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
sinkSequenceBarrier,
new SinkStoreStage());
sinkStoreStage = new BatchEventProcessor<>(disruptorMsgBuffer,
sinkSequenceBarrier,
new SinkStoreStage());
sinkStoreStage.setExceptionHandler(exceptionHandler);
disruptorMsgBuffer.addGatingSequences(sinkStoreStage.getSequence());
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
+4 -4
View File
@@ -88,8 +88,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
private volatile AviaterRegexFilter nameFilter; // 运行时引用可能会有变化，比如规则发生变化时
private volatile AviaterRegexFilter nameBlackFilter;
private Map<String, List<String>> fieldFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldFilterMap = new HashMap<>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<>();
private TableMetaCache tableMetaCache;
private Charset charset = Charset.defaultCharset();
@@ -1040,7 +1040,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (fieldFilterMap != null) {
this.fieldFilterMap = fieldFilterMap;
} else {
this.fieldFilterMap = new HashMap<String, List<String>>();
this.fieldFilterMap = new HashMap<>();
}
for (Map.Entry<String, List<String>> entry : this.fieldFilterMap.entrySet()) {
@@ -1052,7 +1052,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if (fieldBlackFilterMap != null) {
this.fieldBlackFilterMap = fieldBlackFilterMap;
} else {
this.fieldBlackFilterMap = new HashMap<String, List<String>>();
this.fieldBlackFilterMap = new HashMap<>();
}
for (Map.Entry<String, List<String>> entry : this.fieldBlackFilterMap.entrySet()) {
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
+3 -3
View File
@@ -105,7 +105,7 @@ public class TableMetaCache {
TableMeta tableMeta = memoryTableMeta.find(schema, table);
return tableMeta.getFields();
} else {
return new ArrayList<FieldMeta>();
return new ArrayList<>();
}
}
@@ -113,7 +113,7 @@ public class TableMetaCache {
* 处理desc table的结果
*/
public static List<FieldMeta> parseTableMetaByDesc(ResultSetPacket packet) {
Map<String, Integer> nameMaps = new HashMap<String, Integer>(6, 1f);
Map<String, Integer> nameMaps = new HashMap<>(6, 1f);
int index = 0;
for (FieldPacket fieldPacket : packet.getFieldDescriptors()) {
nameMaps.put(fieldPacket.getOriginalName(), index++);
@@ -121,7 +121,7 @@ public class TableMetaCache {
int size = packet.getFieldDescriptors().size();
int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size();
List<FieldMeta> result = new ArrayList<FieldMeta>();
List<FieldMeta> result = new ArrayList<>();
for (int i = 0; i < count; i++) {
FieldMeta meta = new FieldMeta();
// 做一个优化，使用String.intern()，共享String对象，减少内存使用
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java
+1 -1
View File
@@ -56,7 +56,7 @@ public class DruidDdlParser {
return Arrays.asList(ddlResult);
}
List<DdlResult> ddlResults = new ArrayList<DdlResult>();
List<DdlResult> ddlResults = new ArrayList<>();
for (SQLStatement statement : stmtList) {
if (statement instanceof SQLCreateTableStatement) {
DdlResult ddlResult = new DdlResult();
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java
+4 -10
View File
@@ -26,7 +26,7 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException;
public class BinLogFileQueue {
private String baseName = "mysql-bin.";
private List<File> binlogs = new ArrayList<File>();
private List<File> binlogs = new ArrayList<>();
private File directory;
private ReentrantLock lock = new ReentrantLock();
private Condition nextCondition = lock.newCondition();
@@ -177,7 +177,7 @@ public class BinLogFileQueue {
* 获取当前所有binlog文件
*/
public List<File> currentBinlogs() {
return new ArrayList<File>(binlogs);
return new ArrayList<>(binlogs);
}
public void destory() {
@@ -216,7 +216,7 @@ public class BinLogFileQueue {
}
private List<File> listBinlogFiles() {
List<File> files = new ArrayList<File>();
List<File> files = new ArrayList<>();
files.addAll(FileUtils.listFiles(directory, new IOFileFilter() {
public boolean accept(File file) {
@@ -230,13 +230,7 @@ public class BinLogFileQueue {
}
}, null));
// 排一下序列
Collections.sort(files, new Comparator<File>() {
public int compare(File o1, File o2) {
return o1.getName().compareTo(o2.getName());
}
});
Collections.sort(files, Comparator.comparing(File::getName));
return files;
}
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/BinlogDownloadQueue.java
+6 -16
View File
@@ -53,8 +53,8 @@ public class BinlogDownloadQueue {
private static final Logger logger = LoggerFactory.getLogger(BinlogDownloadQueue.class);
private static final int TIMEOUT = 10000;
private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<BinlogFile>();
private LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<Runnable>();
private LinkedBlockingQueue<BinlogFile> downloadQueue = new LinkedBlockingQueue<>();
private LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<>();
private LinkedList<BinlogFile> binlogList;
private final int batchFileSize;
private Thread downloadThread;
@@ -78,13 +78,7 @@ public class BinlogDownloadQueue {
String fileName = StringUtils.substringBetween(binlog.getDownloadLink(), "mysql-bin.", "?");
binlog.setFileName(fileName);
}
Collections.sort(this.binlogList, new Comparator<BinlogFile>() {
@Override
public int compare(BinlogFile o1, BinlogFile o2) {
return o1.getFileName().compareTo(o2.getFileName());
}
});
this.binlogList.sort(Comparator.comparing(BinlogFile::getFileName));
}
public void cleanDir() throws IOException {
@@ -187,13 +181,9 @@ public class BinlogDownloadQueue {
builder.setMaxConnPerRoute(50);
builder.setMaxConnTotal(100);
// 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
final SSLContext sslContext = new SSLContextBuilder()
.loadTrustMaterial(null, (x509Certificates, s) -> true)
.build();
httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext)
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java
+6 -14
View File
@@ -140,13 +140,9 @@ public class HttpHelper {
CloseableHttpResponse response = null;
try {
// 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
final SSLContext sslContext = new SSLContextBuilder()
.loadTrustMaterial(null, (x509Certificates, s) -> true)
.build();
CloseableHttpClient httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext)
@@ -205,13 +201,9 @@ public class HttpHelper {
CloseableHttpResponse response = null;
try {
// 创建支持忽略证书的https
final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return true;
}
}).build();
final SSLContext sslContext = new SSLContextBuilder()
.loadTrustMaterial(null, (x509Certificates, s) -> true)
.build();
CloseableHttpClient httpClient = HttpClientBuilder.create()
.setSSLContext(sslContext)
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogEventParserProxy.java
+31 -54
View File
@@ -30,16 +30,12 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
private int batchFileSize = 4; // 最多下载的binlog文件数量
private RdsLocalBinlogEventParser rdsLocalBinlogEventParser = null;
private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
private ExecutorService executorService = Executors.newSingleThreadExecutor(r -> {
Thread t = new Thread(r,
"rds-binlog-daemon-thread");
t.setDaemon(true);
return t;
}
});
});
@Override
public void start() {
@@ -79,32 +75,17 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
rdsLocalBinlogEventParser.setParallel(this.parallel);
rdsLocalBinlogEventParser.setParallelBufferSize(this.parallelBufferSize);
rdsLocalBinlogEventParser.setParallelThreadSize(this.parallelThreadSize);
rdsLocalBinlogEventParser.setFinishListener(new RdsLocalBinlogEventParser.ParseFinishListener() {
@Override
public void onFinish() {
executorService.execute(new Runnable() {
@Override
public void run() {
rdsLocalBinlogEventParser.stop();
// empty the dump error count,or will go into local binlog mode again,with error
// position,never get out,fixed by bucketli
RdsBinlogEventParserProxy.this.setDumpErrorCount(0);
RdsBinlogEventParserProxy.this.start();
}
});
}
});
this.setParserExceptionHandler(new ParserExceptionHandler() {
@Override
public void handle(Throwable e) {
handleMysqlParserException(e);
if (targetHandler != null) {
targetHandler.handle(e);
}
rdsLocalBinlogEventParser.setFinishListener(() -> executorService.execute(() -> {
rdsLocalBinlogEventParser.stop();
// empty the dump error count,or will go into local binlog mode again,with error
// position,never get out,fixed by bucketli
RdsBinlogEventParserProxy.this.setDumpErrorCount(0);
RdsBinlogEventParserProxy.this.start();
}));
this.setParserExceptionHandler(e -> {
handleMysqlParserException(e);
if (targetHandler != null) {
targetHandler.handle(e);
}
});
}
@@ -115,29 +96,25 @@ public class RdsBinlogEventParserProxy extends MysqlEventParser {
private void handleMysqlParserException(Throwable throwable) {
if (throwable instanceof PositionNotFoundException) {
logger.info("remove rds not found position, try download rds binlog!");
executorService.execute(new Runnable() {
executorService.execute(() -> {
try {
logger.info("stop mysql parser!");
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
long serverId = rdsBinlogEventParserProxy.getServerId();
rdsLocalBinlogEventParser.setServerId(serverId);
rdsBinlogEventParserProxy.stop();
} catch (Throwable e) {
logger.info("handle exception failed", e);
}
@Override
public void run() {
try {
logger.info("stop mysql parser!");
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
long serverId = rdsBinlogEventParserProxy.getServerId();
rdsLocalBinlogEventParser.setServerId(serverId);
rdsBinlogEventParserProxy.stop();
} catch (Throwable e) {
logger.info("handle exception failed", e);
}
try {
logger.info("start rds mysql binlog parser!");
rdsLocalBinlogEventParser.start();
} catch (Throwable e) {
logger.info("handle exception failed", e);
rdsLocalBinlogEventParser.stop();
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
rdsBinlogEventParserProxy.start();// 继续重试
}
try {
logger.info("start rds mysql binlog parser!");
rdsLocalBinlogEventParser.start();
} catch (Throwable e) {
logger.info("handle exception failed", e);
rdsLocalBinlogEventParser.stop();
RdsBinlogEventParserProxy rdsBinlogEventParserProxy = RdsBinlogEventParserProxy.this;
rdsBinlogEventParserProxy.start();// 继续重试
}
});
}
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java
+4 -14
View File
@@ -82,13 +82,7 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
logger.error("download binlog failed", e);
throw new CanalParseException(e);
}
setParserExceptionHandler(new ParserExceptionHandler() {
@Override
public void handle(Throwable e) {
handleMysqlParserException(e);
}
});
setParserExceptionHandler(this::handleMysqlParserException);
super.start();
}
@@ -105,13 +99,9 @@ public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements
}
try {
binlogDownloadQueue.execute(new Runnable() {
@Override
public void run() {
RdsLocalBinlogEventParser.super.stop();
RdsLocalBinlogEventParser.super.start();
}
binlogDownloadQueue.execute(() -> {
RdsLocalBinlogEventParser.super.stop();
RdsLocalBinlogEventParser.super.start();
});
} catch (InterruptedException e) {
throw new RuntimeException(e);
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/request/AbstractRequest.java
+2 -8
View File
@@ -177,7 +177,7 @@ public abstract class AbstractRequest<T> {
private String makeRequestString(Map<String, String> param) throws Exception {
fillCommonParam(param);
String sign = makeSignature(new TreeMap<String, String>(param));
String sign = makeSignature(new TreeMap<>(param));
StringBuilder builder = new StringBuilder();
for (Map.Entry<String, String> entry : param.entrySet()) {
builder.append(encode(entry.getKey())).append("=").append(encode(entry.getValue())).append("&");
@@ -195,13 +195,7 @@ public abstract class AbstractRequest<T> {
*/
@SuppressWarnings("deprecation")
private final HttpResponse executeHttpRequest(HttpGet getMethod, String host) throws Exception {
SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
return true;
}
}).build();
SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, (TrustStrategy) (arg0, arg1) -> true).build();
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext,
new String[] { "TLSv1" },
null,
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java
+26 -34
View File
@@ -52,16 +52,12 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(r,
"[scheduler-table-meta-snapshot]");
thread.setDaemon(true);
return thread;
}
});
private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread thread = new Thread(r,
"[scheduler-table-meta-snapshot]");
thread.setDaemon(true);
return thread;
});
private ReadWriteLock lock = new ReentrantReadWriteLock();
private AtomicBoolean initialized = new AtomicBoolean(false);
private String destination;
@@ -69,8 +65,8 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private volatile MysqlConnection connection; // 查询meta信息的链接
private CanalEventFilter filter;
private CanalEventFilter blackFilter;
private Map<String, List<String>> fieldFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<String, List<String>>();
private Map<String, List<String>> fieldFilterMap = new HashMap<>();
private Map<String, List<String>> fieldBlackFilterMap = new HashMap<>();
private EntryPosition lastPosition;
private boolean hasNewDdl;
private MetaHistoryDAO metaHistoryDAO;
@@ -91,26 +87,22 @@ public class DatabaseTableMeta implements TableMetaTSDB {
// 24小时生成一份snapshot
if (snapshotInterval > 0) {
scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(new Runnable() {
scheduleSnapshotFuture = scheduler.scheduleWithFixedDelay(() -> {
boolean applyResult = false;
try {
MDC.put("destination", destination);
applyResult = applySnapshotToDB(lastPosition, false);
} catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e);
}
@Override
public void run() {
boolean applyResult = false;
try {
MDC.put("destination", destination);
applyResult = applySnapshotToDB(lastPosition, false);
} catch (Throwable e) {
logger.error("scheudle applySnapshotToDB faield", e);
}
try {
MDC.put("destination", destination);
if (applyResult) {
snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
}
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
try {
MDC.put("destination", destination);
if (applyResult) {
snapshotExpire((int) TimeUnit.HOURS.toSeconds(snapshotExpire));
}
} catch (Throwable e) {
logger.error("scheudle snapshotExpire faield", e);
}
}, snapshotInterval, snapshotInterval, TimeUnit.HOURS);
}
@@ -199,7 +191,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter filter) {
try {
ResultSetPacket packet = connection.query("show databases");
List<String> schemas = new ArrayList<String>();
List<String> schemas = new ArrayList<>();
for (String schema : packet.getFieldValues()) {
schemas.add(schema);
}
@@ -207,7 +199,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
for (String schema : schemas) {
// filter views
packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
List<String> tables = new ArrayList<String>();
List<String> tables = new ArrayList<>();
for (String table : packet.getFieldValues()) {
if ("BASE TABLE".equalsIgnoreCase(table)) {
continue;
@@ -245,7 +237,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
}
private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) {
Map<String, String> content = new HashMap<String, String>();
Map<String, String> content = new HashMap<>();
content.put("destination", destination);
content.put("binlogFile", position.getJournalName());
content.put("binlogOffest", String.valueOf(position.getPosition()));
@@ -326,7 +318,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
}
if (compareAll) {
Map<String, String> content = new HashMap<String, String>();
Map<String, String> content = new HashMap<>();
content.put("destination", destination);
content.put("binlogFile", position.getJournalName());
content.put("binlogOffest", String.valueOf(position.getPosition()));
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java
+2 -2
View File
@@ -49,7 +49,7 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
public class MemoryTableMeta implements TableMetaTSDB {
private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class);
private Map<List<String>, TableMeta> tableMetas = new ConcurrentHashMap<List<String>, TableMeta>();
private Map<List<String>, TableMeta> tableMetas = new ConcurrentHashMap<>();
private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
public MemoryTableMeta(){
@@ -143,7 +143,7 @@ public class MemoryTableMeta implements TableMetaTSDB {
}
public Map<String, String> snapshot() {
Map<String, String> schemaDdls = new HashMap<String, String>();
Map<String, String> schemaDdls = new HashMap<>();
for (Schema schema : repository.getSchemas()) {
StringBuffer data = new StringBuffer(4 * 1024);
for (String table : schema.showTables()) {
parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
+12 -20
View File
@@ -68,15 +68,10 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
this.period = period;
this.memoryLogPositionManager = memoryLogPositionManager;
this.dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination) {
return getDataFile(destination);
}
});
this.dataFileCaches = MigrateMap.makeComputingMap(this::getDataFile);
this.executorService = Executors.newScheduledThreadPool(1);
this.persistTasks = Collections.synchronizedSet(new HashSet<String>());
this.persistTasks = Collections.synchronizedSet(new HashSet<>());
}
@Override
@@ -100,19 +95,16 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager {
}
// 启动定时工作任务
executorService.scheduleAtFixedRate(new Runnable() {
public void run() {
List<String> tasks = new ArrayList<String>(persistTasks);
for (String destination : tasks) {
try {
// 定时将内存中的最新值刷到file中，多次变更只刷一次
flushDataToFile(destination);
persistTasks.remove(destination);
} catch (Throwable e) {
// ignore
logger.error("period update" + destination + " curosr failed!", e);
}
executorService.scheduleAtFixedRate(() -> {
List<String> tasks = new ArrayList<>(persistTasks);
for (String destination : tasks) {
try {
// 定时将内存中的最新值刷到file中，多次变更只刷一次
flushDataToFile(destination);
persistTasks.remove(destination);
} catch (Throwable e) {
// ignore
logger.error("period update" + destination + " curosr failed!", e);
}
}
}, period, period, TimeUnit.MILLISECONDS);
parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
+5 -8
View File
@@ -73,14 +73,11 @@ public class MixedLogPositionManager extends AbstractLogPositionManager {
@Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
memoryLogPositionManager.persistLogPosition(destination, logPosition);
executor.submit(new Runnable() {
public void run() {
try {
zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
} catch (Exception e) {
logger.error("ERROR # persist to zookeeper has an error", e);
}
executor.submit(() -> {
try {
zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
} catch (Exception e) {
logger.error("ERROR # persist to zookeeper has an error", e);
}
});
}

Some files were not shown because too many files have changed in this diff Show More