wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
agapple
2017-07-04 21:49:22 +08:00
committed by GitHub
parent 9ba93bae5f ecdd619da7
commit 22e3d2f3f1
86 changed files with 22815 additions and 22428 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7
View File
@@ -80,3 +80,10 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
<li>新浪微博： agapple0002 </li>
<li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
</ol>
<pre>
【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊.
具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
</pre>
client/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java
+4 -4
View File
@@ -108,7 +108,7 @@ public class ClusterCanalConnector implements CanalConnector {
return;
} catch (Throwable t) {
logger.warn("something goes wrong when subscribing from server:{}\n{}",
currentConnector.getAddress(),
currentConnector != null ? currentConnector.getAddress() : "null",
ExceptionUtils.getFullStackTrace(t));
times++;
restart();
@@ -127,7 +127,7 @@ public class ClusterCanalConnector implements CanalConnector {
return;
} catch (Throwable t) {
logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
currentConnector.getAddress(),
currentConnector != null ? currentConnector.getAddress() : "null",
ExceptionUtils.getFullStackTrace(t));
times++;
restart();
@@ -145,7 +145,7 @@ public class ClusterCanalConnector implements CanalConnector {
return msg;
} catch (Throwable t) {
logger.warn("something goes wrong when getting data from server:{}\n{}",
currentConnector.getAddress(),
currentConnector != null ? currentConnector.getAddress() : "null",
ExceptionUtils.getFullStackTrace(t));
times++;
restart();
@@ -163,7 +163,7 @@ public class ClusterCanalConnector implements CanalConnector {
return msg;
} catch (Throwable t) {
logger.warn("something goes wrong when getting data from server:{}\n{}",
currentConnector.getAddress(),
currentConnector != null ? currentConnector.getAddress() : "null",
ExceptionUtils.getFullStackTrace(t));
times++;
restart();
client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java
-1
View File
@@ -114,5 +114,4 @@ public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
return zkClient;
}
}
common/pom.xml
+3 -7
View File
@@ -1,10 +1,9 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>canal.common</artifactId>
@@ -16,16 +15,14 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.6.Final</version>
</dependency>
<!-- zk -->
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
<groupId>com.github.sgroschupf</groupId>
<groupId>com.101tec</groupId>
<artifactId>zkclient</artifactId>
</dependency>
<!-- external -->
@@ -36,7 +33,6 @@
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java
+3 -2
View File
@@ -88,8 +88,9 @@ public class JsonUtils {
public static InetAddressSerializer instance = new InetAddressSerializer();
public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType)
throws IOException {
@Override
public void write(JSONSerializer serializer, Object object, Object fieldName, Type fieldType, int features)
throws IOException {
if (object == null) {
serializer.writeNull();
return;
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java
+37 -98
View File
@@ -1,5 +1,18 @@
package com.alibaba.otter.canal.common.zookeeper;
import org.I0Itec.zkclient.ZkConnection;
import org.I0Itec.zkclient.exception.ZkException;
import org.apache.commons.lang.StringUtils;
import org.apache.zookeeper.ClientCnxn;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.client.ConnectStringParser;
import org.apache.zookeeper.client.HostProvider;
import org.apache.zookeeper.client.StaticHostProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.ReflectionUtils;
import java.io.IOException;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
@@ -8,44 +21,24 @@ import java.util.List;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.I0Itec.zkclient.IZkConnection;
import org.I0Itec.zkclient.exception.ZkException;
import org.apache.commons.lang.StringUtils;
import org.apache.zookeeper.ClientCnxn;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.ZooKeeper.States;
import org.apache.zookeeper.client.ConnectStringParser;
import org.apache.zookeeper.client.HostProvider;
import org.apache.zookeeper.client.StaticHostProvider;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.ReflectionUtils;
/**
* 封装了ZooKeeper，使其支持节点的优先顺序，比如美国机房的节点会优先加载美国对应的zk集群列表，都失败后才会选择加载杭州的zk集群列表 *
*
* @author jianghang 2012-7-10 下午02:31:42
* @version 1.0.0
*/
public class ZooKeeperx implements IZkConnection {
public class ZooKeeperx extends ZkConnection {
private static final String SERVER_COMMA = ";";
private static final Logger logger = LoggerFactory.getLogger(ZooKeeperx.class);
private static final Field clientCnxnField = ReflectionUtils.findField(ZooKeeper.class, "cnxn");
private static final Field hostProviderField = ReflectionUtils.findField(ClientCnxn.class, "hostProvider");
private static final Field serverAddressesField = ReflectionUtils.findField(StaticHostProvider.class,
"serverAddresses");
private static final Field serverAddressesField = ReflectionUtils.findField(StaticHostProvider.class, "serverAddresses");
private static final Field zookeeperLockField = ReflectionUtils.findField(ZkConnection.class, "_zookeeperLock");
private static final Field zookeeperFiled = ReflectionUtils.findField(ZkConnection.class, "_zk");
private static final int DEFAULT_SESSION_TIMEOUT = 90000;
private ZooKeeper _zk = null;
private Lock _zookeeperLock = new ReentrantLock();
private final List<String> _servers;
private final List<String> _serversList;
private final int _sessionTimeOut;
public ZooKeeperx(String zkServers){
@@ -53,114 +46,59 @@ public class ZooKeeperx implements IZkConnection {
}
public ZooKeeperx(String zkServers, int sessionTimeOut){
_servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA));
super(zkServers, sessionTimeOut);
_serversList = Arrays.asList(StringUtils.split(this.getServers(), SERVER_COMMA));
_sessionTimeOut = sessionTimeOut;
}
@Override
public void connect(Watcher watcher) {
ReflectionUtils.makeAccessible(zookeeperLockField);
ReflectionUtils.makeAccessible(zookeeperFiled);
Lock _zookeeperLock = (ReentrantLock) ReflectionUtils.getField(zookeeperLockField, this);
ZooKeeper _zk = (ZooKeeper) ReflectionUtils.getField(zookeeperFiled, this);
_zookeeperLock.lock();
try {
if (_zk != null) {
throw new IllegalStateException("zk client has already been started");
}
String zkServers = _serversList.get(0);
try {
logger.debug("Creating new ZookKeeper instance to connect to " + _servers + ".");
_zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher);
logger.debug("Creating new ZookKeeper instance to connect to " + zkServers + ".");
_zk = new ZooKeeper(zkServers, _sessionTimeOut, watcher);
configMutliCluster(_zk);
ReflectionUtils.setField(zookeeperFiled, this, _zk);
} catch (IOException e) {
throw new ZkException("Unable to connect to " + _servers, e);
throw new ZkException("Unable to connect to " + zkServers, e);
}
} finally {
_zookeeperLock.unlock();
}
}
public void close() throws InterruptedException {
_zookeeperLock.lock();
try {
if (_zk != null) {
logger.debug("Closing ZooKeeper connected to " + _servers);
_zk.close();
_zk = null;
}
} finally {
_zookeeperLock.unlock();
}
}
public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException {
return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode);
}
public void delete(String path) throws InterruptedException, KeeperException {
_zk.delete(path, -1);
}
public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException {
return _zk.exists(path, watch) != null;
}
public List<String> getChildren(final String path, final boolean watch) throws KeeperException,
InterruptedException {
return _zk.getChildren(path, watch);
}
public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException {
return _zk.getData(path, watch, stat);
}
public void writeData(String path, byte[] data) throws KeeperException, InterruptedException {
writeData(path, data, -1);
}
public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException {
_zk.setData(path, data, version);
}
public States getZookeeperState() {
return _zk != null ? _zk.getState() : null;
}
public ZooKeeper getZookeeper() {
return _zk;
}
public long getCreateTime(String path) throws KeeperException, InterruptedException {
Stat stat = _zk.exists(path, false);
if (stat != null) {
return stat.getCtime();
}
return -1;
}
public String getServers() {
return StringUtils.join(_servers, SERVER_COMMA);
}
// ===============================
public void configMutliCluster(ZooKeeper zk) {
if (_servers.size() == 1) {
if (_serversList.size() == 1) {
return;
}
String cluster1 = _servers.get(0);
String cluster1 = _serversList.get(0);
try {
if (_servers.size() > 1) {
if (_serversList.size() > 1) {
// 强制的声明accessible
ReflectionUtils.makeAccessible(clientCnxnField);
ReflectionUtils.makeAccessible(hostProviderField);
ReflectionUtils.makeAccessible(serverAddressesField);
// 添加第二组集群列表
for (int i = 1; i < _servers.size(); i++) {
String cluster = _servers.get(i);
for (int i = 1; i < _serversList.size(); i++) {
String cluster = _serversList.get(i);
// 强制获取zk中的地址信息
ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(serverAddressesField,
hostProvider);
List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(serverAddressesField, hostProvider);
// 添加第二组集群列表
serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());
}
@@ -178,3 +116,4 @@ public class ZooKeeperx implements IZkConnection {
}
}
dbsync/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/JsonConversion.java
+21 -20
View File
@@ -52,23 +52,24 @@ public class JsonConversion {
public static final int VALUE_ENTRY_SIZE_SMALL = (1 + SMALL_OFFSET_SIZE);
public static final int VALUE_ENTRY_SIZE_LARGE = (1 + LARGE_OFFSET_SIZE);
public static Json_Value parse_value(int type, LogBuffer buffer, long len) {
public static Json_Value parse_value(int type, LogBuffer buffer, long len, String charsetName) {
buffer = buffer.duplicate(buffer.position(), (int) len);
switch (type) {
case JSONB_TYPE_SMALL_OBJECT:
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false);
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, false, charsetName);
case JSONB_TYPE_LARGE_OBJECT:
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true);
return parse_array_or_object(Json_enum_type.OBJECT, buffer, len, true, charsetName);
case JSONB_TYPE_SMALL_ARRAY:
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false);
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, false, charsetName);
case JSONB_TYPE_LARGE_ARRAY:
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true);
return parse_array_or_object(Json_enum_type.ARRAY, buffer, len, true, charsetName);
default:
return parse_scalar(type, buffer, len);
return parse_scalar(type, buffer, len, charsetName);
}
}
private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large) {
private static Json_Value parse_array_or_object(Json_enum_type type, LogBuffer buffer, long len, boolean large,
String charsetName) {
long offset_size = large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
if (len < 2 * offset_size) {
throw new IllegalArgumentException("illegal json data");
@@ -95,7 +96,7 @@ public class JsonConversion {
return large ? buffer.getUint32() : buffer.getUint16();
}
private static Json_Value parse_scalar(int type, LogBuffer buffer, long len) {
private static Json_Value parse_scalar(int type, LogBuffer buffer, long len, String charsetName) {
switch (type) {
case JSONB_TYPE_LITERAL:
/* purecov: inspected */
@@ -146,12 +147,12 @@ public class JsonConversion {
}
}
if (str_len == 0 || len < n + str_len) {
if (len < n + str_len) {
throw new IllegalArgumentException("illegal json data");
}
return new Json_Value(Json_enum_type.STRING, buffer.rewind()
.forward((int) n)
.getFixString((int) str_len));
.getFixString((int) str_len, charsetName));
case JSONB_TYPE_OPAQUE:
/*
* There should always be at least one byte, which tells the
@@ -237,7 +238,7 @@ public class JsonConversion {
this.m_large = large;
}
public String key(int i) {
public String key(int i, String charsetName) {
m_data.rewind();
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
@@ -250,10 +251,10 @@ public class JsonConversion {
// entry, always two
// bytes.
long key_length = m_data.getUint16();
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length);
return m_data.rewind().forward((int) key_offset).getFixString((int) key_length, charsetName);
}
public Json_Value element(int i) {
public Json_Value element(int i, String charsetName) {
m_data.rewind();
int offset_size = m_large ? LARGE_OFFSET_SIZE : SMALL_OFFSET_SIZE;
int key_entry_size = m_large ? KEY_ENTRY_SIZE_LARGE : KEY_ENTRY_SIZE_SMALL;
@@ -266,13 +267,13 @@ public class JsonConversion {
int type = m_data.forward(entry_offset).getUint8();
if (type == JSONB_TYPE_INT16 || type == JSONB_TYPE_UINT16 || type == JSONB_TYPE_LITERAL
|| (m_large && (type == JSONB_TYPE_INT32 || type == JSONB_TYPE_UINT32))) {
return parse_scalar(type, m_data, value_entry_size - 1);
return parse_scalar(type, m_data, value_entry_size - 1, charsetName);
}
int value_offset = (int) read_offset_or_size(m_data, m_large);
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset);
return parse_value(type, m_data.rewind().forward(value_offset), (int) m_length - value_offset, charsetName);
}
public StringBuilder toJsonString(StringBuilder buf) {
public StringBuilder toJsonString(StringBuilder buf, String charsetName) {
switch (m_type) {
case OBJECT:
buf.append("{");
@@ -280,9 +281,9 @@ public class JsonConversion {
if (i > 0) {
buf.append(", ");
}
buf.append('"').append(key(i)).append('"');
buf.append('"').append(key(i, charsetName)).append('"');
buf.append(": ");
element(i).toJsonString(buf);
element(i, charsetName).toJsonString(buf, charsetName);
}
buf.append("}");
break;
@@ -292,7 +293,7 @@ public class JsonConversion {
if (i > 0) {
buf.append(", ");
}
element(i).toJsonString(buf);
element(i, charsetName).toJsonString(buf, charsetName);
}
buf.append("]");
break;
@@ -361,7 +362,7 @@ public class JsonConversion {
}
buf.append('"').append(text).append('"');
} else {
text = m_data.getFixString((int) m_length);
text = m_data.getFixString((int) m_length, charsetName);
buf.append('"').append(text).append('"');
}
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java
+2 -2
View File
@@ -168,8 +168,8 @@ public class LogBuffer {
* @param newLimit The new limit value; must be non-negative and no larger
* than this buffer's capacity
* @return This buffer
* @throws IllegalArgumentException If the preconditions on <tt>newLimit</tt>
* do not hold
* @throws IllegalArgumentException If the preconditions on
* <tt>newLimit</tt> do not hold
*/
public final LogBuffer limit(int newLimit) {
if (origin + newLimit > buffer.length || newLimit < 0) throw new IllegalArgumentException("capacity excceed: "
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java
+6 -6
View File
@@ -140,15 +140,15 @@ public final class FormatDescriptionLogEvent extends StartLogEventV3 {
throw new IOException("Unknown binlog version: " + binlogVersion);
}
}
public FormatDescriptionLogEvent(final int binlogVersion,int binlogChecksum){
this(binlogVersion);
this.header.checksumAlg = binlogChecksum;
public FormatDescriptionLogEvent(final int binlogVersion, int binlogChecksum){
this(binlogVersion);
this.header.checksumAlg = binlogChecksum;
}
public FormatDescriptionLogEvent(final int binlogVersion){
this.binlogVersion = binlogVersion;
postHeaderLen = new short[ENUM_END_EVENT];
/* identify binlog format */
switch (binlogVersion) {
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java
+49 -33
View File
@@ -330,17 +330,14 @@ public class QueryLogEvent extends LogEvent {
* MariaDb type,
* sec_part of NOW()
*/
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 16 + 1 + 60/*
* type
* ,
* user_len
* ,
* user
* ,
* host_len
* ,
* host
*/);
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + 1 + 32 * 3
+ 1 + 60/*
* type ,
* user_len ,
* user ,
* host_len ,
* host
*/);
/**
* Fixed data part:
* <ul>
@@ -397,7 +394,7 @@ public class QueryLogEvent extends LogEvent {
// inspection by the DBA
private final long execTime;
private final int errorCode;
private final long sessionId; /* thread_id */
private final long sessionId; /* thread_id */
/**
* 'flags2' is a second set of flags (on top of those in Log_event), for
@@ -499,26 +496,26 @@ public class QueryLogEvent extends LogEvent {
}
/* query event post-header */
public static final int Q_THREAD_ID_OFFSET = 0;
public static final int Q_EXEC_TIME_OFFSET = 4;
public static final int Q_DB_LEN_OFFSET = 8;
public static final int Q_ERR_CODE_OFFSET = 9;
public static final int Q_STATUS_VARS_LEN_OFFSET = 11;
public static final int Q_DATA_OFFSET = QUERY_HEADER_LEN;
public static final int Q_THREAD_ID_OFFSET = 0;
public static final int Q_EXEC_TIME_OFFSET = 4;
public static final int Q_DB_LEN_OFFSET = 8;
public static final int Q_ERR_CODE_OFFSET = 9;
public static final int Q_STATUS_VARS_LEN_OFFSET = 11;
public static final int Q_DATA_OFFSET = QUERY_HEADER_LEN;
/* these are codes, not offsets; not more than 256 values (1 byte). */
public static final int Q_FLAGS2_CODE = 0;
public static final int Q_SQL_MODE_CODE = 1;
public static final int Q_FLAGS2_CODE = 0;
public static final int Q_SQL_MODE_CODE = 1;
/**
* Q_CATALOG_CODE is catalog with end zero stored; it is used only by MySQL
* 5.0.x where 0<=x<=3. We have to keep it to be able to replicate these old
* masters.
*/
public static final int Q_CATALOG_CODE = 2;
public static final int Q_AUTO_INCREMENT = 3;
public static final int Q_CHARSET_CODE = 4;
public static final int Q_TIME_ZONE_CODE = 5;
public static final int Q_CATALOG_CODE = 2;
public static final int Q_AUTO_INCREMENT = 3;
public static final int Q_CHARSET_CODE = 4;
public static final int Q_TIME_ZONE_CODE = 5;
/**
* Q_CATALOG_NZ_CODE is catalog withOUT end zero stored; it is used by MySQL
@@ -528,31 +525,47 @@ public class QueryLogEvent extends LogEvent {
* would crash (segfault etc) because it would expect a 0 when there is
* none.
*/
public static final int Q_CATALOG_NZ_CODE = 6;
public static final int Q_CATALOG_NZ_CODE = 6;
public static final int Q_LC_TIME_NAMES_CODE = 7;
public static final int Q_LC_TIME_NAMES_CODE = 7;
public static final int Q_CHARSET_DATABASE_CODE = 8;
public static final int Q_CHARSET_DATABASE_CODE = 8;
public static final int Q_TABLE_MAP_FOR_UPDATE_CODE = 9;
public static final int Q_TABLE_MAP_FOR_UPDATE_CODE = 9;
public static final int Q_MASTER_DATA_WRITTEN_CODE = 10;
public static final int Q_MASTER_DATA_WRITTEN_CODE = 10;
public static final int Q_INVOKER = 11;
public static final int Q_INVOKER = 11;
/**
* Q_UPDATED_DB_NAMES status variable collects of the updated databases
* total number and their names to be propagated to the slave in order to
* facilitate the parallel applying of the Query events.
*/
public static final int Q_UPDATED_DB_NAMES = 12;
public static final int Q_UPDATED_DB_NAMES = 12;
public static final int Q_MICROSECONDS = 13;
public static final int Q_MICROSECONDS = 13;
/**
* A old (unused now) code for Query_log_event status similar to
* G_COMMIT_TS.
*/
public static final int Q_COMMIT_TS = 14;
/**
* A code for Query_log_event status, similar to G_COMMIT_TS2.
*/
public static final int Q_COMMIT_TS2 = 15;
/**
* The master connection @@session.explicit_defaults_for_timestamp which is
* recorded for queries, CREATE and ALTER table that is defined with a
* TIMESTAMP column, that are dependent on that feature. For pre-WL6292
* master's the associated with this code value is zero.
*/
public static final int Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP = 16;
/**
* FROM MariaDB 5.5.34
*/
public static final int Q_HRNOW = 128;
public static final int Q_HRNOW = 128;
private final void unpackVariables(LogBuffer buffer, final int end) throws IOException {
int code = -1;
@@ -632,6 +645,9 @@ public class QueryLogEvent extends LogEvent {
mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
}
break;
case Q_EXPLICIT_DEFAULTS_FOR_TIMESTAMP:
buffer.forward(1);
break;
case Q_HRNOW:
// int when_sec_part = buffer.getUint24();
buffer.forward(3);
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java
+1 -1
View File
@@ -90,7 +90,7 @@ public final class RotateLogEvent extends LogEvent {
int filenameLen = buffer.limit() - filenameOffset;
if (filenameLen > FN_REFLEN - 1) filenameLen = FN_REFLEN - 1;
buffer.position(filenameOffset);
filename = buffer.getFixString(filenameLen);
}
dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java
+2 -2
View File
@@ -952,9 +952,9 @@ public final class RowsLogBuffer {
len = buffer.getUint16();
buffer.forward(meta - 2);
int position = buffer.position();
Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1);
Json_Value jsonValue = JsonConversion.parse_value(buffer.getUint8(), buffer, len - 1, charsetName);
StringBuilder builder = new StringBuilder();
jsonValue.toJsonString(builder);
jsonValue.toJsonString(builder, charsetName);
value = builder.toString();
buffer.position(position + len);
// byte[] binary = new byte[len];
deployer/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+10
View File
@@ -149,6 +149,11 @@ public class CanalController {
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
} finally {
@@ -383,6 +388,11 @@ public class CanalController {
public void handleNewSession() throws Exception {
initCid(path);
}
@Override
public void handleSessionEstablishmentError(Throwable error) throws Exception {
logger.error("failed to connect to zookeeper", error);
}
});
}
// 优先启动embeded服务
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java
+21 -2
View File
@@ -49,6 +49,8 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan"));
private volatile boolean isFirst = true;
public void start() {
super.start();
Assert.notNull(rootConf, "root conf dir is null!");
@@ -58,6 +60,9 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
public void run() {
try {
scan();
if (isFirst) {
isFirst = false;
}
} catch (Throwable e) {
logger.error("scan failed", e);
}
@@ -122,13 +127,18 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
if (!actions.containsKey(destination) && instanceConfigs.length > 0) {
// 存在合法的instance.properties，并且第一次添加时，进行启动操作
notifyStart(instanceDir, destination);
notifyStart(instanceDir, destination, instanceConfigs);
} else if (actions.containsKey(destination)) {
// 历史已经启动过
if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties
notifyStop(destination);
} else {
InstanceConfigFiles lastFile = lastFiles.get(destination);
// 历史启动过 所以配置文件信息必然存在
if (!isFirst && CollectionUtils.isEmpty(lastFile.getInstanceFiles())) {
logger.error("[{}] is started, but not found instance file info.", destination);
}
boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles());
// 通知变化
if (hasChanged) {
@@ -161,10 +171,19 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
}
}
private void notifyStart(File instanceDir, String destination) {
private void notifyStart(File instanceDir, String destination, File[] instanceConfigs) {
try {
defaultAction.start(destination);
actions.put(destination, defaultAction);
// 启动成功后记录配置文件信息
InstanceConfigFiles lastFile = lastFiles.get(destination);
List<FileInfo> newFileInfo = new ArrayList<FileInfo>();
for (File instanceConfig : instanceConfigs) {
newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified()));
}
lastFile.setInstanceFiles(newFileInfo);
logger.info("auto notify start {} successful.", destination);
} catch (Throwable e) {
logger.error("scan add found[{}] but start failed", destination, ExceptionUtils.getFullStackTrace(e));
deployer/src/main/resources/logback.xml
+2 -4
View File
@@ -74,10 +74,8 @@
<appender-ref ref="CANAL-META" />
</logger>
<root level="WARN">
<!--
<appender-ref ref="STDOUT"/>
-->
<root level="INFO">
<!--<appender-ref ref="STDOUT"/>-->
<appender-ref ref="CANAL-ROOT" />
</root>
</configuration>
deployer/src/main/resources/spring/default-instance.xml
+5 -5
View File
@@ -128,14 +128,14 @@
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
<constructor-arg ref="metaManager"/>
</bean>
</property>
</constructor-arg>
</bean>
</property>
deployer/src/main/resources/spring/file-instance.xml
+5 -5
View File
@@ -113,14 +113,14 @@
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
<constructor-arg ref="metaManager"/>
</bean>
</property>
</constructor-arg>
</bean>
</property>
deployer/src/main/resources/spring/local-instance.xml
+5 -5
View File
@@ -123,14 +123,14 @@
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
<constructor-arg ref="metaManager"/>
</bean>
</property>
</constructor-arg>
</bean>
</property>
driver/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java
+3 -1
View File
@@ -65,8 +65,10 @@ public class MysqlConnector {
public void connect() throws IOException {
if (connected.compareAndSet(false, true)) {
try {
channel = SocketChannelPool.open(address);
channel = SocketChannel.open();
configChannel(channel);
logger.info("connect MysqlConnection to {}...", address);
channel.connect(address);
negotiate(channel);
} catch (Exception e) {
disconnect();
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java
+3 -3
View File
@@ -24,9 +24,9 @@ public class MysqlQueryExecutor {
private SocketChannel channel;
public MysqlQueryExecutor(MysqlConnector connector){
public MysqlQueryExecutor(MysqlConnector connector) throws IOException{
if (!connector.isConnected()) {
throw new RuntimeException("should execute connector.connect() first");
throw new IOException("should execute connector.connect() first");
}
this.channel = connector.getChannel();
@@ -88,7 +88,7 @@ public class MysqlQueryExecutor {
for (RowDataPacket r : rowData) {
resultSet.getFieldValues().addAll(r.getColumns());
}
resultSet.setSourceAddress(channel.getRemoteSocketAddress());
resultSet.setSourceAddress(channel.socket().getRemoteSocketAddress());
return resultSet;
}
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java
+2 -2
View File
@@ -23,9 +23,9 @@ public class MysqlUpdateExecutor {
private MysqlConnector connector;
public MysqlUpdateExecutor(MysqlConnector connector){
public MysqlUpdateExecutor(MysqlConnector connector) throws IOException{
if (!connector.isConnected()) {
throw new RuntimeException("should execute connector.connect() first");
throw new IOException("should execute connector.connect() first");
}
this.connector = connector;
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java
+1 -1
View File
@@ -74,7 +74,7 @@ public class FieldPacket extends PacketWithHeaderPacket {
this.decimals = data[index];
index++;
//
index += 2;//skip filter
index += 2;// skip filter
//
if (index < data.length) {
reader.setIndex(index);
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java
+2 -2
View File
@@ -9,9 +9,9 @@ import java.io.IOException;
import java.net.SocketAddress;
/**
* 封装netty的通信channel和数据接收缓存，实现读、写、连接校验的功能。 2016-12-28
*
* @author luoyaogui
* 封装netty的通信channel和数据接收缓存，实现读、写、连接校验的功能。
* 2016-12-28
*/
public class SocketChannel {
private Channel channel = null;
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java
+3 -4
View File
@@ -20,12 +20,11 @@ import java.io.IOException;
import java.net.SocketAddress;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author luoyaogui
* 实现channel的管理（监听连接、读数据、回收）
* 2016-12-28
* @author luoyaogui 实现channel的管理（监听连接、读数据、回收） 2016-12-28
*/
@SuppressWarnings("rawtypes")
@SuppressWarnings({ "rawtypes", "deprecation" })
public abstract class SocketChannelPool {
private static EventLoopGroup group = new NioEventLoopGroup();//非阻塞IO线程组
driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java
-2
View File
@@ -1,9 +1,7 @@
package com.alibaba.otter.canal.parse.driver.mysql.utils;
import java.io.IOException;
import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
import com.alibaba.otter.canal.parse.driver.mysql.socket.SocketChannel;
public abstract class PacketManager {
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java
-1
View File
@@ -4,7 +4,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
example/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
filter/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java
-1
View File
@@ -1,7 +1,6 @@
package com.alibaba.otter.canal.filter;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.filter.aviater.AviaterELFilter;
filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java
+1 -2
View File
@@ -5,9 +5,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Assert;
import org.apache.commons.lang.RandomStringUtils;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
instance/core/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>canal.instance.core</artifactId>
instance/core/src/main/java/com/alibaba/otter/canal/instance/core/AbstractCanalInstance.java
+16 -19
View File
@@ -1,5 +1,12 @@
package com.alibaba.otter.canal.instance.core;
import java.util.List;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
@@ -10,36 +17,27 @@ import com.alibaba.otter.canal.parse.ha.HeartBeatHAController;
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.canal.store.CanalEventStore;
import com.alibaba.otter.canal.store.model.Event;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
/**
* Created with Intellig IDEA.
* Author: yinxiu
* Date: 2016-01-07
* Time: 22:26
* Created with Intellig IDEA. Author: yinxiu Date: 2016-01-07 Time: 22:26
*/
public class AbstractCanalInstance extends AbstractCanalLifeCycle implements CanalInstance {
private static final Logger logger = LoggerFactory.getLogger(AbstractCanalInstance.class);
private static final Logger logger = LoggerFactory.getLogger(AbstractCanalInstance.class);
protected Long canalId; // 和manager交互唯一标示
protected String destination; // 队列名字
protected CanalEventStore<Event> eventStore; // 有序队列
protected Long canalId; // 和manager交互唯一标示
protected String destination; // 队列名字
protected CanalEventStore<Event> eventStore; // 有序队列
protected CanalEventParser eventParser; // 解析对应的数据信息
protected CanalEventSink<List<CanalEntry.Entry>> eventSink; // 链接parse和store的桥接器
protected CanalMetaManager metaManager; // 消费信息管理器
protected CanalAlarmHandler alarmHandler; // alarm报警机制
protected CanalEventParser eventParser; // 解析对应的数据信息
protected CanalEventSink<List<CanalEntry.Entry>> eventSink; // 链接parse和store的桥接器
protected CanalMetaManager metaManager; // 消费信息管理器
protected CanalAlarmHandler alarmHandler; // alarm报警机制
@Override
public boolean subscribeChange(ClientIdentity identity) {
@@ -121,7 +119,6 @@ public class AbstractCanalInstance extends AbstractCanalLifeCycle implements Can
alarmHandler.stop();
}
logger.info("stop successful....");
}
instance/manager/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
+12 -22
View File
@@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import com.alibaba.otter.canal.parse.index.*;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,12 +38,6 @@ import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
import com.alibaba.otter.canal.sink.entry.EntryEventSink;
@@ -346,29 +341,24 @@ public class CanalInstanceWithManager extends AbstractCanalInstance {
protected CanalLogPositionManager initLogPositionManager() {
logger.info("init logPositionPersistManager begin...");
IndexMode indexMode = parameters.getIndexMode();
CanalLogPositionManager logPositionManager = null;
CanalLogPositionManager logPositionManager;
if (indexMode.isMemory()) {
logPositionManager = new MemoryLogPositionManager();
} else if (indexMode.isZookeeper()) {
logPositionManager = new ZooKeeperLogPositionManager();
((ZooKeeperLogPositionManager) logPositionManager).setZkClientx(getZkclientx());
logPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
} else if (indexMode.isMixed()) {
logPositionManager = new PeriodMixedLogPositionManager();
ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager();
zooKeeperLogPositionManager.setZkClientx(getZkclientx());
((PeriodMixedLogPositionManager) logPositionManager).setZooKeeperLogPositionManager(zooKeeperLogPositionManager);
MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
logPositionManager = new PeriodMixedLogPositionManager(memoryLogPositionManager,
zooKeeperLogPositionManager,
1000L);
} else if (indexMode.isMeta()) {
logPositionManager = new MetaLogPositionManager();
((MetaLogPositionManager) logPositionManager).setMetaManager(metaManager);
logPositionManager = new MetaLogPositionManager(metaManager);
} else if (indexMode.isMemoryMetaFailback()) {
MemoryLogPositionManager primaryLogPositionManager = new MemoryLogPositionManager();
MetaLogPositionManager failbackLogPositionManager = new MetaLogPositionManager();
failbackLogPositionManager.setMetaManager(metaManager);
MemoryLogPositionManager primary = new MemoryLogPositionManager();
MetaLogPositionManager secondary = new MetaLogPositionManager(metaManager);
logPositionManager = new FailbackLogPositionManager();
((FailbackLogPositionManager) logPositionManager).setPrimary(primaryLogPositionManager);
((FailbackLogPositionManager) logPositionManager).setFailback(failbackLogPositionManager);
logPositionManager = new FailbackLogPositionManager(primary, secondary);
} else {
throw new CanalException("unsupport indexMode for " + indexMode);
}
instance/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
instance/spring/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
+2 -2
View File
@@ -2,11 +2,11 @@ package com.alibaba.otter.canal.instance.spring;
import java.util.List;
import com.alibaba.otter.canal.instance.core.AbstractCanalInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
import com.alibaba.otter.canal.instance.core.AbstractCanalInstance;
import com.alibaba.otter.canal.meta.CanalMetaManager;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.protocol.CanalEntry;
@@ -23,7 +23,7 @@ import com.alibaba.otter.canal.store.model.Event;
*/
public class CanalInstanceWithSpring extends AbstractCanalInstance {
private static final Logger logger = LoggerFactory.getLogger(CanalInstanceWithSpring.class);
private static final Logger logger = LoggerFactory.getLogger(CanalInstanceWithSpring.class);
public void start() {
logger.info("start CannalInstance for {}-{} ", new Object[] { 1, destination });
instance/spring/src/test/resources/spring/default-instance.xml
+5 -5
View File
@@ -120,14 +120,14 @@
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
<constructor-arg ref="metaManager"/>
</bean>
</property>
</constructor-arg>
</bean>
</property>
instance/spring/src/test/resources/spring/file-instance.xml
+8 -6
View File
@@ -102,18 +102,20 @@
<!-- 解析编码 -->
<!-- property name="connectionCharsetNumber" value="${canal.instance.connectionCharsetNumber:33}" /-->
<property name="connectionCharset" value="${canal.instance.connectionCharset:UTF-8}" />
<!-- 解析位点记录 -->
<property name="logPositionManager">
<bean class="com.alibaba.otter.canal.parse.index.FailbackLogPositionManager">
<property name="primary">
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MemoryLogPositionManager" />
</property>
<property name="failback">
</constructor-arg>
<constructor-arg>
<bean class="com.alibaba.otter.canal.parse.index.MetaLogPositionManager">
<property name="metaManager" ref="metaManager" />
<constructor-arg>
<ref bean="metaManager"/>
</constructor-arg>
</bean>
</property>
</constructor-arg>
</bean>
</property>
meta/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java
+1 -2
View File
@@ -5,9 +5,8 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java
-1
View File
@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.meta;
import java.util.Map;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.protocol.position.PositionRange;
meta/src/test/java/com/alibaba/otter/canal/meta/MixedMetaManagerTest.java
+1 -2
View File
@@ -3,9 +3,8 @@ package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
meta/src/test/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManagerTest.java
+1 -2
View File
@@ -3,9 +3,8 @@ package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
meta/src/test/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManagerTest.java
+1 -2
View File
@@ -2,9 +2,8 @@ package com.alibaba.otter.canal.meta;
import java.util.Map;
import org.junit.Assert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
parse/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>canal.parse</artifactId>
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
+3
View File
@@ -191,6 +191,9 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
} catch (TableIdNotFoundException e) {
throw e;
} catch (Throwable e) {
if (e.getCause() instanceof TableIdNotFoundException) {
throw (TableIdNotFoundException) e.getCause();
}
// 记录一下，出错的位点信息
processSinkError(e,
this.lastPosition,
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
+19 -22
View File
@@ -83,30 +83,27 @@ public class LocalBinLogConnection implements ErosaConnection {
context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
while (running) {
boolean needContinue = true;
LogEvent event = null;
L:while (fetcher.fetch()) {
/*event = decoder.decode(fetcher, context);
if (event == null) {
throw new CanalParseException("parse failed");
}
LogEvent event = null;
L: while (fetcher.fetch()) {
/*
* event = decoder.decode(fetcher, context); if (event ==
* null) { throw new CanalParseException("parse failed"); }
* if (!func.sink(event)) { needContinue = false; break; }
*/
if (!func.sink(event)) {
needContinue = false;
break;
}*/
do {
if (event == null) {
event = new RotateLogEvent(context.getLogPosition().getFileName(), context.getLogPosition()
.getPosition());
} else {
event = decoder.decode(fetcher, context);
}
do {
if (event == null) {
event = new RotateLogEvent(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
} else {
event = decoder.decode(fetcher, context);
}
if (event != null && !func.sink(event)) {
needContinue = false;
break L;
}
} while (event != null);
if (event != null && !func.sink(event)) {
needContinue = false;
break L;
}
} while (event != null);
}
if (needContinue) {// 读取下一个
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
+18 -18
View File
@@ -27,7 +27,7 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
private EntryPosition masterPosition; // binlog信息
private MysqlConnection metaConnection; // 查询meta信息的链接
private TableMetaCache tableMetaCache; // 对应meta
private String directory;
private boolean needWait = false;
private int bufferSize = 16 * 1024;
@@ -39,24 +39,24 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
@Override
protected ErosaConnection buildErosaConnection() {
return buildLocalBinLogConnection();
}
}
@Override
protected void preDump(ErosaConnection connection) {
metaConnection = buildMysqlConnection();
protected void preDump(ErosaConnection connection) {
metaConnection = buildMysqlConnection();
try {
metaConnection.connect();
} catch (IOException e) {
throw new CanalParseException(e);
}
tableMetaCache = new TableMetaCache(metaConnection);
((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}
}
@Override
protected void afterDump(ErosaConnection connection) {
if (metaConnection != null) {
@Override
protected void afterDump(ErosaConnection connection) {
if (metaConnection != null) {
try {
metaConnection.disconnect();
} catch (IOException e) {
@@ -64,19 +64,19 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
.getAddress(), e);
}
}
}
}
public void start() throws CanalParseException {
public void start() throws CanalParseException {
if (runningInfo == null) { // 第一次链接主库
runningInfo = masterInfo;
}
super.start();
}
@Override
public void stop() {
if (metaConnection != null) {
public void stop() {
if (metaConnection != null) {
try {
metaConnection.disconnect();
} catch (IOException e) {
@@ -90,9 +90,9 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
}
super.stop();
}
}
private ErosaConnection buildLocalBinLogConnection() {
private ErosaConnection buildLocalBinLogConnection() {
LocalBinLogConnection connection = new LocalBinLogConnection();
connection.setBufferSize(this.bufferSize);
@@ -114,7 +114,7 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements
connection.setCharset(connectionCharset);
return connection;
}
@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
// 处理逻辑
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
+10 -1
View File
@@ -139,7 +139,6 @@ public class MysqlConnection implements ErosaConnection {
binlogDumpHeader.setPacketBodyLength(cmdBody.length);
binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(),cmdBody);
connector.setDumping(true);
}
@@ -199,6 +198,16 @@ public class MysqlConnection implements ErosaConnection {
logger.warn(ExceptionUtils.getFullStackTrace(e));
}
try {
// 参考:https://github.com/alibaba/canal/issues/284
// mysql5.6需要设置slave_uuid避免被server kill链接
update("set @slave_uuid=uuid()");
} catch (Exception e) {
if (!StringUtils.contains(e.getMessage(), "Unknown system variable")) {
logger.warn(ExceptionUtils.getFullStackTrace(e));
}
}
try {
// mariadb针对特殊的类型，需要设置session变量
update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java
+3 -3
View File
@@ -190,9 +190,9 @@ public class BinLogFileQueue {
files.addAll(FileUtils.listFiles(directory, new IOFileFilter() {
public boolean accept(File file) {
Pattern pattern = Pattern.compile("\\d+$");
Matcher matcher = pattern.matcher(file.getName());
return file.getName().startsWith(baseName) && matcher.find();
Pattern pattern = Pattern.compile("\\d+$");
Matcher matcher = pattern.matcher(file.getName());
return file.getName().startsWith(baseName) && matcher.find();
}
public boolean accept(File dir, String name) {
parse/src/main/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManager.java
+9
View File
@@ -0,0 +1,9 @@
package com.alibaba.otter.canal.parse.index;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
/**
* Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com
*/
public abstract class AbstractLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java
+2 -4
View File
@@ -5,14 +5,12 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
/**
* 接口组合
*
* @author jianghang 2012-7-7 上午10:02:02
* @version 1.0.0
* Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com
*/
public interface CanalLogPositionManager extends CanalLifeCycle {
LogPosition getLatestIndexBy(String destination);
void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException;
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/FailbackLogPositionManager.java
+38 -29
View File
@@ -1,74 +1,83 @@
package com.alibaba.otter.canal.parse.index;
import org.springframework.util.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
/**
* Created by yinxiu on 17/3/18. Email: marklin.hz@gmail.com
* 实现基于failover查找的机制完成meta的操作
*
*
* <pre>
* 应用场景：比如针对内存buffer，出现HA切换，先尝试从内存buffer区中找到lastest position，如果不存在才尝试找一下meta里消费的信息
* </pre>
*
* @author jianghang 2012-7-20 下午02:33:20
*/
public class FailbackLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
public class FailbackLogPositionManager extends AbstractLogPositionManager {
private CanalLogPositionManager primary;
private CanalLogPositionManager failback;
private final static Logger logger = LoggerFactory.getLogger(FailbackLogPositionManager.class);
private final CanalLogPositionManager primary;
private final CanalLogPositionManager secondary;
public FailbackLogPositionManager(CanalLogPositionManager primary, CanalLogPositionManager secondary){
if (primary == null) {
throw new NullPointerException("nul primary LogPositionManager");
}
if (secondary == null) {
throw new NullPointerException("nul secondary LogPositionManager");
}
this.primary = primary;
this.secondary = secondary;
}
@Override
public void start() {
super.start();
Assert.notNull(primary);
Assert.notNull(failback);
if (!primary.isStart()) {
primary.start();
}
if (!failback.isStart()) {
failback.start();
if (!secondary.isStart()) {
secondary.start();
}
}
@Override
public void stop() {
super.stop();
if (secondary.isStart()) {
secondary.stop();
}
if (primary.isStart()) {
primary.stop();
}
if (failback.isStart()) {
failback.stop();
}
}
@Override
public LogPosition getLatestIndexBy(String destination) {
LogPosition logPosition = primary.getLatestIndexBy(destination);
if (logPosition == null) {
return failback.getLatestIndexBy(destination);
} else {
if (logPosition != null) {
return logPosition;
}
return secondary.getLatestIndexBy(destination);
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
try {
primary.persistLogPosition(destination, logPosition);
} catch (CanalParseException e) {
failback.persistLogPosition(destination, logPosition);
logger.warn("persistLogPosition use primary log position manager exception. destination: {}, logPosition: {}",
destination,
logPosition,
e);
secondary.persistLogPosition(destination, logPosition);
}
}
public void setPrimary(CanalLogPositionManager primary) {
this.primary = primary;
}
public void setFailback(CanalLogPositionManager failback) {
this.failback = failback;
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
+67 -62
View File
@@ -16,45 +16,72 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
/**
* 基于文件刷新的log position实现
*
* Created by yinxiu on 17/3/18. Email: marklin.hz@gmail.com 基于文件刷新的log
* position实现
*
* <pre>
* 策略：
* 1. 先写内存，然后定时刷新数据到File
* 2. 数据采取overwrite模式(只保留最后一次)
* </pre>
*
* @author jianghang 2013-4-15 下午09:40:48
* @version 1.0.4
*/
public class FileMixedLogPositionManager extends MemoryLogPositionManager {
public class FileMixedLogPositionManager extends AbstractLogPositionManager {
private final static Logger logger = LoggerFactory.getLogger(FileMixedLogPositionManager.class);
private final static Charset charset = Charset.forName("UTF-8");
private static final Logger logger = LoggerFactory.getLogger(FileMixedLogPositionManager.class);
private static final Charset charset = Charset.forName("UTF-8");
private File dataDir;
private String dataFileName = "parse.dat";
private Map<String, File> dataFileCaches;
private ScheduledExecutorService executor;
@SuppressWarnings("serial")
private ScheduledExecutorService executorService;
private final LogPosition nullPosition = new LogPosition() {
};
private long period = 1000; // 单位ms
private MemoryLogPositionManager memoryLogPositionManager;
private long period;
private Set<String> persistTasks;
public FileMixedLogPositionManager(File dataDir, long period, MemoryLogPositionManager memoryLogPositionManager){
if (dataDir == null) {
throw new NullPointerException("null dataDir");
}
if (period <= 0) {
throw new IllegalArgumentException("period must be positive, given: " + period);
}
if (memoryLogPositionManager == null) {
throw new NullPointerException("null memoryLogPositionManager");
}
this.dataDir = dataDir;
this.period = period;
this.memoryLogPositionManager = memoryLogPositionManager;
this.dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination) {
return getDataFile(destination);
}
});
this.executorService = Executors.newScheduledThreadPool(1);
this.persistTasks = Collections.synchronizedSet(new HashSet<String>());
}
@Override
public void start() {
super.start();
Assert.notNull(dataDir);
if (!dataDir.exists()) {
try {
FileUtils.forceMkdir(dataDir);
@@ -67,30 +94,12 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {
public File apply(String destination) {
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {
public LogPosition apply(String destination) {
LogPosition logPosition = loadDataFromFile(dataFileCaches.get(destination));
if (logPosition == null) {
return nullPosition;
} else {
return logPosition;
}
}
});
persistTasks = Collections.synchronizedSet(new HashSet<String>());
if (!memoryLogPositionManager.isStart()) {
memoryLogPositionManager.start();
}
// 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() {
executorService.scheduleAtFixedRate(new Runnable() {
public void run() {
List<String> tasks = new ArrayList<String>(persistTasks);
@@ -106,28 +115,35 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
}
}
}, period, period, TimeUnit.MILLISECONDS);
}
@Override
public void stop() {
super.stop();
flushDataToFile();
executor.shutdownNow();
positions.clear();
}
public void persistLogPosition(String destination, LogPosition logPosition) {
persistTasks.add(destination);// 添加到任务队列中进行触发
super.persistLogPosition(destination, logPosition);
executorService.shutdown();
memoryLogPositionManager.stop();
}
@Override
public LogPosition getLatestIndexBy(String destination) {
LogPosition logPostion = super.getLatestIndexBy(destination);
if (logPostion == nullPosition) {
return null;
} else {
return logPostion;
LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
if (logPosition != null) {
return logPosition;
}
logPosition = loadDataFromFile(dataFileCaches.get(destination));
if (logPosition == null) {
return nullPosition;
}
return logPosition;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
persistTasks.add(destination);
memoryLogPositionManager.persistLogPosition(destination, logPosition);
}
// ============================ helper method ======================
@@ -142,11 +158,12 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
}
}
String dataFileName = "parse.dat";
return new File(destinationMetaDir, dataFileName);
}
private void flushDataToFile() {
for (String destination : positions.keySet()) {
for (String destination : memoryLogPositionManager.destinations()) {
flushDataToFile(destination);
}
}
@@ -156,7 +173,7 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
}
private void flushDataToFile(String destination, File dataFile) {
LogPosition position = positions.get(destination);
LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
if (position != null && position != nullPosition) {
String json = JsonUtils.marshalToString(position);
try {
@@ -179,16 +196,4 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
throw new CanalMetaManagerException(e);
}
}
public void setDataDir(String dataDir) {
this.dataDir = new File(dataDir);
}
public void setDataDir(File dataDir) {
this.dataDir = dataDir;
}
public void setPeriod(long period) {
this.period = period;
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java
+14 -10
View File
@@ -1,39 +1,43 @@
package com.alibaba.otter.canal.parse.index;
import java.util.Map;
import java.util.Set;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.collect.MapMaker;
/**
* 基于内存的实现
*
* @author jianghang 2012-7-7 上午10:17:23
* @version 1.0.0
* Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com
*/
public class MemoryLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
public class MemoryLogPositionManager extends AbstractLogPositionManager {
protected Map<String, LogPosition> positions;
private Map<String, LogPosition> positions;
@Override
public void start() {
super.start();
positions = new MapMaker().makeMap();
}
@Override
public void stop() {
super.stop();
positions.clear();
}
@Override
public LogPosition getLatestIndexBy(String destination) {
return positions.get(destination);
}
public void persistLogPosition(String destination, LogPosition logPosition) {
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
positions.put(destination, logPosition);
}
public Set<String> destinations() {
return positions.keySet();
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManager.java
+28 -23
View File
@@ -4,52 +4,56 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.meta.CanalMetaManager;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.store.helper.CanalEventUtils;
/**
* 基于meta信息的实现
*
* @author jianghang 2012-7-10 下午05:02:33
* @version 1.0.0
* Created by yinxiu on 17/3/18. Email: marklin.hz@gmail.com
*/
public class MetaLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
public class MetaLogPositionManager extends AbstractLogPositionManager {
private static final Logger logger = LoggerFactory.getLogger(MetaLogPositionManager.class);
private CanalMetaManager metaManager;
private final static Logger logger = LoggerFactory.getLogger(MetaLogPositionManager.class);
public void start() {
super.start();
Assert.notNull(metaManager);
if (!metaManager.isStart()) {
metaManager.start();
private final CanalMetaManager metaManager;
public MetaLogPositionManager(CanalMetaManager metaManager){
if (metaManager == null) {
throw new NullPointerException("null metaManager");
}
this.metaManager = metaManager;
}
@Override
public void stop() {
super.stop();
if (metaManager.isStart()) {
metaManager.stop();
}
}
public void persistLogPosition(String destination, LogPosition logPosition) {
// do nothing
logger.info("persist LogPosition:{}", destination, logPosition);
@Override
public void start() {
super.start();
if (!metaManager.isStart()) {
metaManager.start();
}
}
@Override
public LogPosition getLatestIndexBy(String destination) {
List<ClientIdentity> clientIdentitys = metaManager.listAllSubscribeInfo(destination);
List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
LogPosition result = null;
if (!CollectionUtils.isEmpty(clientIdentitys)) {
if (!CollectionUtils.isEmpty(clientIdentities)) {
// 尝试找到一个最小的logPosition
for (ClientIdentity clientIdentity : clientIdentitys) {
for (ClientIdentity clientIdentity : clientIdentities) {
LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
if (position == null) {
continue;
@@ -66,8 +70,9 @@ public class MetaLogPositionManager extends AbstractCanalLifeCycle implements Ca
return result;
}
public void setMetaManager(CanalMetaManager metaManager) {
this.metaManager = metaManager;
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
// do nothing
logger.info("persist LogPosition:{}", destination, logPosition);
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
+49 -52
View File
@@ -5,86 +5,83 @@ import java.util.concurrent.Executors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
/**
* 混合memory + zookeeper的存储模式
*
* @author jianghang 2012-7-7 上午10:33:19
* @version 1.0.0
* Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com Memory first.
* Asynchronous commit position info to ZK.
*/
public class MixedLogPositionManager extends MemoryLogPositionManager implements CanalLogPositionManager {
public class MixedLogPositionManager extends AbstractLogPositionManager {
private static final Logger logger = LoggerFactory.getLogger(MixedLogPositionManager.class);
private ZooKeeperLogPositionManager zooKeeperLogPositionManager;
private ExecutorService executor;
@SuppressWarnings("serial")
private final LogPosition nullPosition = new LogPosition() {
};
private final Logger logger = LoggerFactory.getLogger(MixedLogPositionManager.class);
private final MemoryLogPositionManager memoryLogPositionManager;
private final ZooKeeperLogPositionManager zooKeeperLogPositionManager;
private final ExecutorService executor;
public MixedLogPositionManager(ZkClientx zkClient){
if (zkClient == null) {
throw new NullPointerException("null zkClient");
}
this.memoryLogPositionManager = new MemoryLogPositionManager();
this.zooKeeperLogPositionManager = new ZooKeeperLogPositionManager(zkClient);
this.executor = Executors.newFixedThreadPool(1);
}
@Override
public void start() {
super.start();
Assert.notNull(zooKeeperLogPositionManager);
if (!memoryLogPositionManager.isStart()) {
memoryLogPositionManager.start();
}
if (!zooKeeperLogPositionManager.isStart()) {
zooKeeperLogPositionManager.start();
}
executor = Executors.newFixedThreadPool(1);
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {
public LogPosition apply(String destination) {
LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
if (logPosition == null) {
return nullPosition;
} else {
return logPosition;
}
}
});
}
@Override
public void stop() {
super.stop();
if (zooKeeperLogPositionManager.isStart()) {
zooKeeperLogPositionManager.stop();
}
executor.shutdownNow();
positions.clear();
executor.shutdown();
zooKeeperLogPositionManager.stop();
memoryLogPositionManager.stop();
}
public void persistLogPosition(final String destination, final LogPosition logPosition) {
super.persistLogPosition(destination, logPosition);
@Override
public LogPosition getLatestIndexBy(String destination) {
LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
if (logPosition != null) {
return logPosition;
}
logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
// 这里保持和重构前的逻辑一致,重新添加到Memory中
if (logPosition != null) {
memoryLogPositionManager.persistLogPosition(destination, logPosition);
}
return logPosition;
}
@Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
memoryLogPositionManager.persistLogPosition(destination, logPosition);
executor.submit(new Runnable() {
public void run() {
try {
zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
} catch (Exception e) {
logger.error("ERROR # persist to zookeepr has an error", e);
logger.error("ERROR # persist to zookeeper has an error", e);
}
}
});
}
public LogPosition getLatestIndexBy(String destination) {
LogPosition logPosition = super.getLatestIndexBy(destination);
if (logPosition == nullPosition) {
return null;
} else {
return logPosition;
}
}
// ======================== setter / getter ======================
public void setZooKeeperLogPositionManager(ZooKeeperLogPositionManager zooKeeperLogPositionManager) {
this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java
+59 -53
View File
@@ -11,54 +11,77 @@ import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.google.common.base.Function;
import com.google.common.collect.MigrateMap;
/**
* 基于定时刷新的策略的mixed实现
*
* @author jianghang 2012-9-12 上午11:18:14
* @version 1.0.0
* Created by yinxiu on 17/3/18. Email: marklin.hz@gmail.com
*/
public class PeriodMixedLogPositionManager extends MemoryLogPositionManager implements CanalLogPositionManager {
public class PeriodMixedLogPositionManager extends AbstractLogPositionManager {
private static final Logger logger = LoggerFactory.getLogger(PeriodMixedLogPositionManager.class);
private MemoryLogPositionManager memoryLogPositionManager;
private ZooKeeperLogPositionManager zooKeeperLogPositionManager;
private ScheduledExecutorService executor;
@SuppressWarnings("serial")
private ScheduledExecutorService executorService;
private long period;
private Set<String> persistTasks;
private final LogPosition nullPosition = new LogPosition() {
};
private long period = 1000; // 单位ms
private Set<String> persistTasks;
public PeriodMixedLogPositionManager(MemoryLogPositionManager memoryLogPositionManager,
ZooKeeperLogPositionManager zooKeeperLogPositionManager, long period){
if (memoryLogPositionManager == null) {
throw new NullPointerException("null memoryLogPositionManager");
}
if (zooKeeperLogPositionManager == null) {
throw new NullPointerException("null zooKeeperLogPositionManager");
}
if (period <= 0) {
throw new IllegalArgumentException("period must be positive, given: " + period);
}
this.memoryLogPositionManager = memoryLogPositionManager;
this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
this.period = period;
this.persistTasks = Collections.synchronizedSet(new HashSet<String>());
this.executorService = Executors.newScheduledThreadPool(1);
}
@Override
public void stop() {
super.stop();
if (zooKeeperLogPositionManager.isStart()) {
zooKeeperLogPositionManager.stop();
}
if (memoryLogPositionManager.isStart()) {
memoryLogPositionManager.stop();
}
executorService.shutdown();
}
@Override
public void start() {
super.start();
Assert.notNull(zooKeeperLogPositionManager);
if (!memoryLogPositionManager.isStart()) {
memoryLogPositionManager.start();
}
if (!zooKeeperLogPositionManager.isStart()) {
zooKeeperLogPositionManager.start();
}
executor = Executors.newScheduledThreadPool(1);
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {
public LogPosition apply(String destination) {
LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
if (logPosition == null) {
return nullPosition;
} else {
return logPosition;
}
}
});
persistTasks = Collections.synchronizedSet(new HashSet<String>());
// 启动定时工作任务
executor.scheduleAtFixedRate(new Runnable() {
executorService.scheduleAtFixedRate(new Runnable() {
public void run() {
List<String> tasks = new ArrayList<String>(persistTasks);
@@ -76,36 +99,19 @@ public class PeriodMixedLogPositionManager extends MemoryLogPositionManager impl
}, period, period, TimeUnit.MILLISECONDS);
}
public void stop() {
super.stop();
if (zooKeeperLogPositionManager.isStart()) {
zooKeeperLogPositionManager.stop();
}
executor.shutdownNow();
positions.clear();
}
public void persistLogPosition(String destination, LogPosition logPosition) {
persistTasks.add(destination);// 添加到任务队列中进行触发
super.persistLogPosition(destination, logPosition);
}
@Override
public LogPosition getLatestIndexBy(String destination) {
LogPosition logPostion = super.getLatestIndexBy(destination);
if (logPostion == nullPosition) {
LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
if (logPosition == nullPosition) {
return null;
} else {
return logPostion;
return logPosition;
}
}
public void setZooKeeperLogPositionManager(ZooKeeperLogPositionManager zooKeeperLogPositionManager) {
this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
persistTasks.add(destination);
memoryLogPositionManager.persistLogPosition(destination, logPosition);
}
public void setPeriod(long period) {
this.period = period;
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManager.java
+12 -21
View File
@@ -1,33 +1,29 @@
package com.alibaba.otter.canal.parse.index;
import org.I0Itec.zkclient.exception.ZkNoNodeException;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.common.utils.JsonUtils;
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.protocol.position.LogPosition;
/**
* 基于zk的实现
*
* @author jianghang 2012-7-7 上午10:08:27
* @version 1.0.0
* Created by yinxiu on 17/3/17. Email: marklin.hz@gmail.com
*/
public class ZooKeeperLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
public class ZooKeeperLogPositionManager extends AbstractLogPositionManager {
private ZkClientx zkClientx;
private final ZkClientx zkClientx;
public void start() {
super.start();
Assert.notNull(zkClientx);
}
public ZooKeeperLogPositionManager(ZkClientx zkClient){
if (zkClient == null) {
throw new NullPointerException("null zkClient");
}
public void stop() {
super.stop();
this.zkClientx = zkClient;
}
@Override
public LogPosition getLatestIndexBy(String destination) {
String path = ZookeeperPathUtils.getParsePath(destination);
byte[] data = zkClientx.readData(path, true);
@@ -38,7 +34,8 @@ public class ZooKeeperLogPositionManager extends AbstractCanalLifeCycle implemen
return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
public void persistLogPosition(String destination, LogPosition logPosition) {
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
String path = ZookeeperPathUtils.getParsePath(destination);
byte[] data = JsonUtils.marshalToByte(logPosition);
try {
@@ -48,10 +45,4 @@ public class ZooKeeperLogPositionManager extends AbstractCanalLifeCycle implemen
}
}
// ================== setter / getter =================
public void setZkClientx(ZkClientx zkClientx) {
this.zkClientx = zkClientx;
}
}
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java
-1
View File
@@ -6,7 +6,6 @@ import java.util.Date;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java
+8 -6
View File
@@ -8,7 +8,7 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.inbound.AbstractBinlogParser;
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
@@ -70,15 +70,17 @@ public class GroupEventPaserTest {
mysqlEventPaser.setMasterPosition(defaultPosition);
mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
mysqlEventPaser.setEventSink(new EntryEventSink());
mysqlEventPaser.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
// System.out.println(logPosition);
}
mysqlEventPaser.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
return mysqlEventPaser;
}
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java
+8 -7
View File
@@ -5,11 +5,11 @@ import java.nio.charset.Charset;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
@@ -81,16 +81,17 @@ public class LocalBinlogDumpTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
controller.start();
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java
+4 -4
View File
@@ -12,8 +12,8 @@ import org.junit.Before;
import org.junit.Test;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.position.EntryPosition;
@@ -70,7 +70,7 @@ public class LocalBinlogEventParserTest {
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
@@ -133,7 +133,7 @@ public class LocalBinlogEventParserTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
@@ -198,7 +198,7 @@ public class LocalBinlogEventParserTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
+8 -7
View File
@@ -5,11 +5,11 @@ import java.nio.charset.Charset;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Column;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
@@ -84,16 +84,17 @@ public class MysqlDumpTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
controller.start();
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java
+11 -9
View File
@@ -8,9 +8,10 @@ import java.util.concurrent.atomic.AtomicLong;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.parse.exception.CanalParseException;
import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager;
import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
@@ -67,16 +68,17 @@ public class MysqlEventParserTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
}
controller.setLogPositionManager(new AbstractLogPositionManager() {
@Override
public LogPosition getLatestIndexBy(String destination) {
return null;
}
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
System.out.println(logPosition);
}
});
controller.start();
@@ -136,7 +138,7 @@ public class MysqlEventParserTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
@@ -205,7 +207,7 @@ public class MysqlEventParserTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
@@ -281,7 +283,7 @@ public class MysqlEventParserTest {
}
});
controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
controller.setLogPositionManager(new AbstractLogPositionManager() {
public void persistLogPosition(String destination, LogPosition logPosition) {
System.out.println(logPosition);
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java
+1 -1
View File
@@ -164,7 +164,7 @@ public class SimpleDdlParserTest {
Assert.assertEquals("retl_mark1", result.getOriTableName());
Assert.assertEquals("retl_mark3", result.getTableName());
//正则匹配test case
// 正则匹配test case
queryString = "rename table totl_mark to totl_mark2";
result = SimpleDdlParser.parse(queryString, "retl");
parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java
+9 -8
View File
@@ -3,9 +3,8 @@ package com.alibaba.otter.canal.parse.index;
import java.io.File;
import java.io.IOException;
import org.junit.Assert;
import org.apache.commons.io.FileUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -27,17 +26,19 @@ public class FileMixedLogPositionManagerTest extends AbstractLogPositionManagerT
@Test
public void testAll() {
FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager();
logPositionManager.setDataDir(dataDir);
logPositionManager.setPeriod(100);
MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager(dataDir,
1000,
memoryLogPositionManager);
logPositionManager.start();
LogPosition position2 = doTest(logPositionManager);
sleep(1500);
FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager();
logPositionManager2.setDataDir(dataDir);
logPositionManager2.setPeriod(100);
FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager(dataDir,
1000,
memoryLogPositionManager);
logPositionManager2.start();
LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java
+2 -4
View File
@@ -3,9 +3,8 @@ package com.alibaba.otter.canal.parse.index;
import java.net.InetSocketAddress;
import java.util.Date;
import org.junit.Assert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -46,8 +45,7 @@ public class MetaLogPositionManagerTest extends AbstractLogPositionManagerTest {
metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
metaManager.start();
MetaLogPositionManager logPositionManager = new MetaLogPositionManager();
logPositionManager.setMetaManager(metaManager);
MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
logPositionManager.start();
// 构建meta信息
ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java
+5 -9
View File
@@ -1,8 +1,7 @@
package com.alibaba.otter.canal.parse.index;
import org.junit.Assert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -28,19 +27,16 @@ public class MixedLogPositionManagerTest extends AbstractLogPositionManagerTest
@Test
public void testAll() {
MixedLogPositionManager logPositionManager = new MixedLogPositionManager();
MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);
ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
zookeeperLogPositionManager.setZkClientx(zkclientx);
logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
logPositionManager.start();
LogPosition position2 = doTest(logPositionManager);
sleep(1000);
MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager();
logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
logPositionManager2.start();
LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java
+5 -8
View File
@@ -1,8 +1,7 @@
package com.alibaba.otter.canal.parse.index;
import org.junit.Assert;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -28,19 +27,17 @@ public class PeriodMixedLogPositionManagerTest extends AbstractLogPositionManage
@Test
public void testAll() {
PeriodMixedLogPositionManager logPositionManager = new PeriodMixedLogPositionManager();
MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);
ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
zookeeperLogPositionManager.setZkClientx(zkclientx);
PeriodMixedLogPositionManager logPositionManager = new PeriodMixedLogPositionManager(memoryLogPositionManager, zookeeperLogPositionManager, 1000L);
logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
logPositionManager.start();
LogPosition position2 = doTest(logPositionManager);
sleep(1500);
PeriodMixedLogPositionManager logPositionManager2 = new PeriodMixedLogPositionManager();
logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
PeriodMixedLogPositionManager logPositionManager2 = new PeriodMixedLogPositionManager(memoryLogPositionManager, zookeeperLogPositionManager, 1000L);
logPositionManager2.start();
LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java
+1 -2
View File
@@ -25,8 +25,7 @@ public class ZooKeeperLogPositionManagerTest extends AbstractLogPositionManagerT
@Test
public void testAll() {
ZooKeeperLogPositionManager logPositionManager = new ZooKeeperLogPositionManager();
logPositionManager.setZkClientx(zkclientx);
ZooKeeperLogPositionManager logPositionManager = new ZooKeeperLogPositionManager(zkclientx);
logPositionManager.start();
doTest(logPositionManager);
parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java
-8
View File
@@ -1,8 +0,0 @@
package com.alibaba.otter.canal.parse.stub;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
public abstract class AbstractCanalLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
}
pom.xml
+10 -5
View File
@@ -4,7 +4,7 @@
<artifactId>canal</artifactId>
<packaging>pom</packaging>
<name>canal module for otter ${project.version}</name>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<url>https://github.com/alibaba/canal</url>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -170,14 +170,14 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.github.sgroschupf</groupId>
<groupId>com.101tec</groupId>
<artifactId>zkclient</artifactId>
<version>0.1</version>
<version>0.10</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.35</version>
<version>1.2.28</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -199,6 +199,11 @@
<artifactId>netty</artifactId>
<version>3.2.5.Final</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.6.Final</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -336,7 +341,7 @@
<keywords>true</keywords>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jxr-plugin</artifactId>
protocol/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
+12553 -12352
View File
File diff suppressed because it is too large Load Diff
protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java
+9595 -9400
View File
File diff suppressed because it is too large Load Diff
server/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>canal.server</artifactId>
server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
+2 -2
View File
@@ -131,8 +131,8 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
*/
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkStart(clientIdentity.getDestination());
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
if (!canalInstance.getMetaManager().isStart()) {
canalInstance.getMetaManager().start();
sink/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
store/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.0.24-SNAPSHOT</version>
<version>1.0.25-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java
+1 -2
View File
@@ -7,10 +7,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.junit.Assert;
import org.apache.commons.lang.math.RandomUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.util.CollectionUtils;