Merge canal repo master
This commit is contained in:
duhengforever
@@ -1,14 +1,6 @@
|
||||
<div class="blog_content">
|
||||
<div class="iteye-blog-content-contain">
|
||||
|
||||
<h3>最新更新</h3>
|
||||
<ol>
|
||||
<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
|
||||
<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
|
||||
<li>Canal已在阿里云推出商业化版本 <a href="https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7">数据传输服务DTS</a>， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。<a href="https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E">DTS产品使用文档</a></li>
|
||||
DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，<a href="https://common-buy.aliyun.com/?commodityCode=dtspre&request=%7b%22dts_function%22%3a%22data_subscribe%22%7d">立即体验>>></a>
|
||||
</ol>
|
||||
|
||||
|
||||
<h1>背景</h1>
|
||||
<p style="font-size: 14px;"> 早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&消费的业务，从此开启了一段新纪元。</p>
|
||||
<p style="font-size: 14px;"> ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.7及以下的版本(阿里内部mysql 5.7.13, 5.6.10, mysql 5.5.18和5.1.40/48)</p>
|
||||
@@ -46,24 +38,54 @@ DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体
|
||||
<li>canal解析binary log对象(原始为byte流)</li>
|
||||
</ol>
|
||||
|
||||
<h1>重要版本更新说明</h1>
|
||||
|
||||
canal 1.1.x系列，参考release文档：<a href="https://github.com/alibaba/canal/releases">版本发布信息</a>
|
||||
|
||||
1. 整体性能测试&优化,提升了150%. #726 参考: 【[Performance](https://github.com/alibaba/canal/wiki/Performance)】
|
||||
2. 原生支持prometheus监控 #765 【[Prometheus QuickStart](https://github.com/alibaba/canal/wiki/Prometheus-QuickStart)】
|
||||
3. 原生支持kafka消息投递 #695 【[Canal Kafka QuickStart](https://github.com/alibaba/canal/wiki/Canal-Kafka-QuickStart)】
|
||||
4. 原生支持aliyun rds的binlog订阅 (解决自动主备切换/oss binlog离线解析) 参考: 【[Aliyun RDS QuickStart](https://github.com/alibaba/canal/wiki/aliyun-RDS-QuickStart)】
|
||||
5. 原生支持docker镜像 #801 参考: 【[Docker QuickStart](https://github.com/alibaba/canal/wiki/Docker-QuickStart)】
|
||||
|
||||
<h1>相关文档</h1>
|
||||
|
||||
See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文档</a>
|
||||
|
||||
<h3><a name="table-of-contents" class="anchor" href="#table-of-contents"><span class="mini-icon mini-icon-link"></span></a>wiki文档列表</h3>
|
||||
<ul>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Home">Home</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Introduction">Introduction</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/QuickStart">QuickStart</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientExample">ClientExample</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/AdminGuide">AdminGuide</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientAPI">ClientAPI</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/DevGuide">DevGuide</a></li>
|
||||
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/BinlogChange%28mysql5.6%29">BinlogChange(Mysql5.6)</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/Home">Home</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/Introduction">Introduction</a></li>
|
||||
<li>
|
||||
<a class="internal present" href="/alibaba/canal/wiki/QuickStart">QuickStart</a>
|
||||
<ul>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/Docker-QuickStart">Docker QuickStart</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/Canal-Kafka-QuickStart">Canal Kafka QuickStart</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/aliyun-RDS-QuickStart">Aliyun RDS QuickStart</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/Prometheus-QuickStart">Prometheus QuickStart</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/AdminGuide">AdminGuide</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/ClientExample">ClientExample</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/ClientAPI">ClientAPI</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/Performance">Performance</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/DevGuide">DevGuide</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/BinlogChange%28mysql5.6%29">BinlogChange(Mysql5.6)</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/BinlogChange%28MariaDB%29">BinlogChange(MariaDB)</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/TableMetaTSDB">TableMetaTSDB</a></li>
|
||||
<li><a href="http://alibaba.github.com/canal/release.html">ReleaseNotes</a></li>
|
||||
<li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
|
||||
<li><a class="internal present" href="/alibaba/canal/wiki/FAQ">FAQ</a></li>
|
||||
</ul>
|
||||
|
||||
<h1>多语言业务</h1>
|
||||
|
||||
1. canal整体交互协议设计上使用了protobuf3.0，理论上可以支持绝大部分的多语言场景，欢迎大家提交多客户端的PR
|
||||
* canal java客户端: <a href="https://github.com/alibaba/canal/wiki/ClientExample"> https://github.com/alibaba/canal/wiki/ClientExample </a>
|
||||
* canal c#客户端开源项目地址：<a href="https://github.com/CanalSharp/CanalSharp"> https://github.com/CanalSharp/CanalSharp </a>
|
||||
* canal go客户端，开发进行中
|
||||
2. canal作为MySQL binlog的增量获取工具，可以将数据投递到MQ系统中，比如Kafka/RocketMQ，可以借助于MQ的多语言能力
|
||||
|
||||
<h1>相关资料</h1>
|
||||
|
||||
* ADC阿里技术嘉年华分享ppt (放在google docs上，可能需要翻墙): <a href="https://docs.google.com/presentation/d/1MkszUPYRDkfVPz9IqOT1LLT5d9tuwde_WC8GZvjaDRg/edit?usp=sharing">ppt下载</href>
|
||||
@@ -91,9 +113,13 @@ See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文
|
||||
<li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
|
||||
</ol>
|
||||
|
||||
<pre>
|
||||
【招聘】阿里巴巴中间件团队招聘JAVA高级工程师
|
||||
岗位主要为技术型内容(非业务部门)，阿里中间件整个体系对于未来想在技术上有所沉淀的同学还是非常有帮助的
|
||||
工作地点：杭州、北京均可. ps. 阿里待遇向来都是不错的，有意者可以QQ、微博私聊.
|
||||
具体招聘内容：https://job.alibaba.com/zhaopin/position_detail.htm?positionId=32666
|
||||
</pre>
|
||||
<h3>最新更新</h3>
|
||||
<ol>
|
||||
<li>canal发布重大版本更新1.1.0，具体releaseNode参考：<a href="https://github.com/alibaba/canal/releases/tag/canal-1.1.0">https://github.com/alibaba/canal/releases/tag/canal-1.1.0</a></li>
|
||||
<li>canal c#客户端开源项目地址：<a href="https://github.com/CanalSharp/CanalSharp"> https://github.com/CanalSharp/CanalSharp </a>，推荐! </li>
|
||||
<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
|
||||
<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
|
||||
|
||||
<li>Canal已在阿里云推出商业化版本 <a href="https://www.aliyun.com/product/dts?spm=a2c4g.11186623.cloudEssentials.80.srdwr7">数据传输服务DTS</a>， 开通即用，免去部署维护的昂贵使用成本。DTS针对阿里云RDS、DRDS等产品进行了适配，解决了Binlog日志回收，主备切换、VPC网络切换等场景下的订阅高可用问题。同时，针对RDS进行了针对性的性能优化。出于稳定性、性能及成本的考虑，强烈推荐阿里云用户使用DTS产品。<a href="https://help.aliyun.com/document_detail/26592.html?spm=a2c4g.11174283.6.539.t1Y91E">DTS产品使用文档</a></li>
|
||||
DTS支持阿里云RDS&DRDS的Binlog日志实时订阅，现推出首月免费体验，限时限量，<a href="https://common-buy.aliyun.com/?commodityCode=dtspre&request=%7b%22dts_function%22%3a%22data_subscribe%22%7d">立即体验>>></a>
|
||||
</ol>
|
||||
+6
-2
@@ -1,7 +1,9 @@
|
||||
package com.alibaba.otter.canal.client.adapter;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
@@ -23,9 +25,11 @@ public interface CanalOuterAdapter {
|
||||
/**
|
||||
* 往适配器中写入数据
|
||||
*
|
||||
* @param message message数据包
|
||||
* @param dml 数据包
|
||||
*/
|
||||
void writeOut(Message message);
|
||||
void writeOut(Dml dml);
|
||||
|
||||
// void writeOut(FlatMessage flatMessage);
|
||||
|
||||
/**
|
||||
* 外部适配器销毁接口
|
||||
|
||||
+11
-19
@@ -22,6 +22,9 @@ public class CanalClientConfig {
|
||||
|
||||
private List<MQTopic> mqTopics;
|
||||
|
||||
private Boolean flatMessage = true;
|
||||
|
||||
|
||||
private List<CanalInstance> canalInstances;
|
||||
|
||||
public String getCanalServerHost() {
|
||||
@@ -60,6 +63,14 @@ public class CanalClientConfig {
|
||||
return mqTopics;
|
||||
}
|
||||
|
||||
public Boolean getFlatMessage() {
|
||||
return flatMessage;
|
||||
}
|
||||
|
||||
public void setFlatMessage(Boolean flatMessage) {
|
||||
this.flatMessage = flatMessage;
|
||||
}
|
||||
|
||||
public void setMqTopics(List<MQTopic> mqTopics) {
|
||||
this.mqTopics = mqTopics;
|
||||
}
|
||||
@@ -166,25 +177,6 @@ public class CanalClientConfig {
|
||||
this.outAdapters = outAdapters;
|
||||
}
|
||||
|
||||
// public List<Adaptor> getAdapters() {
|
||||
// return adapters;
|
||||
// }
|
||||
//
|
||||
// public void setAdapters(List<Adaptor> adapters) {
|
||||
// this.adapters = adapters;
|
||||
// }
|
||||
}
|
||||
|
||||
// public static class Adaptor {
|
||||
// private List<CanalOuterAdapterConfiguration> outAdapters;
|
||||
//
|
||||
// public List<CanalOuterAdapterConfiguration> getOutAdapters() {
|
||||
// return outAdapters;
|
||||
// }
|
||||
//
|
||||
// public void setOutAdapters(List<CanalOuterAdapterConfiguration> outAdapters)
|
||||
// {
|
||||
// this.outAdapters = outAdapters;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
+2
-2
@@ -21,7 +21,7 @@ public class JdbcTypeUtil {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
|
||||
|
||||
public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
|
||||
public static Object typeConvert(String columnName, String value, int sqlType, String mysqlType) {
|
||||
if (value == null || value.equals("")) {
|
||||
return null;
|
||||
}
|
||||
@@ -96,7 +96,7 @@ public class JdbcTypeUtil {
|
||||
}
|
||||
return res;
|
||||
} catch (Exception e) {
|
||||
logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
|
||||
logger.error("table: {} column: {}, failed convert type {} to {}", columnName, value, sqlType);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
+54
-4
@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
|
||||
import java.util.*;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
@@ -63,8 +64,7 @@ public class MessageUtil {
|
||||
|
||||
for (CanalEntry.Column column : columns) {
|
||||
row.put(column.getName(),
|
||||
JdbcTypeUtil.typeConvert(dml.getTable(),
|
||||
column.getName(),
|
||||
JdbcTypeUtil.typeConvert(column.getName(),
|
||||
column.getValue(),
|
||||
column.getSqlType(),
|
||||
column.getMysqlType()));
|
||||
@@ -82,8 +82,7 @@ public class MessageUtil {
|
||||
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
|
||||
if (updateSet.contains(column.getName())) {
|
||||
rowOld.put(column.getName(),
|
||||
JdbcTypeUtil.typeConvert(dml.getTable(),
|
||||
column.getName(),
|
||||
JdbcTypeUtil.typeConvert(column.getName(),
|
||||
column.getValue(),
|
||||
column.getSqlType(),
|
||||
column.getMysqlType()));
|
||||
@@ -106,6 +105,57 @@ public class MessageUtil {
|
||||
}
|
||||
}
|
||||
|
||||
public static Dml flatMessage2Dml(FlatMessage flatMessage) {
|
||||
if (flatMessage == null) {
|
||||
return null;
|
||||
}
|
||||
Dml dml = new Dml();
|
||||
dml.setDatabase(flatMessage.getDatabase());
|
||||
dml.setTable(flatMessage.getTable());
|
||||
dml.setType(flatMessage.getType());
|
||||
dml.setTs(flatMessage.getTs());
|
||||
dml.setSql(flatMessage.getSql());
|
||||
if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
|
||||
throw new RuntimeException("SqlType or mysqlType is null");
|
||||
}
|
||||
List<Map<String, String>> data = flatMessage.getData();
|
||||
if (data != null) {
|
||||
dml.setData(changeRows(data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
|
||||
}
|
||||
List<Map<String, String>> old = flatMessage.getOld();
|
||||
if (old != null) {
|
||||
dml.setOld(changeRows(old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
|
||||
}
|
||||
return dml;
|
||||
}
|
||||
|
||||
private static List<Map<String, Object>> changeRows(List<Map<String, String>> rows, Map<String, Integer> sqlTypes,
|
||||
Map<String, String> mysqlTypes) {
|
||||
List<Map<String, Object>> result = new ArrayList<>();
|
||||
for (Map<String, String> row : rows) {
|
||||
Map<String, Object> resultRow = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, String> entry : row.entrySet()) {
|
||||
String columnName = entry.getKey();
|
||||
String columnValue = entry.getValue();
|
||||
|
||||
Integer sqlType = sqlTypes.get(columnName);
|
||||
if (sqlType == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String mysqlType = mysqlTypes.get(columnName);
|
||||
if (mysqlType == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Object finalValue = JdbcTypeUtil.typeConvert(columnName, columnValue, sqlType, mysqlType);
|
||||
resultRow.put(columnName, finalValue);
|
||||
}
|
||||
result.add(resultRow);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public interface Consumer<T> {
|
||||
|
||||
void accept(T t);
|
||||
|
||||
+9
-14
@@ -4,6 +4,7 @@ import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
@@ -86,20 +87,14 @@ public class HbaseAdapter implements CanalOuterAdapter {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeOut(Message message) {
|
||||
MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
|
||||
|
||||
@Override
|
||||
public void accept(Dml dml) {
|
||||
if (dml == null) {
|
||||
return;
|
||||
}
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
MappingConfig config = mappingConfigCache.get(database + "-" + table);
|
||||
hbaseSyncService.sync(config, dml);
|
||||
}
|
||||
});
|
||||
public void writeOut(Dml dml) {
|
||||
if (dml == null) {
|
||||
return;
|
||||
}
|
||||
String database = dml.getDatabase();
|
||||
String table = dml.getTable();
|
||||
MappingConfig config = mappingConfigCache.get(database + "-" + table);
|
||||
hbaseSyncService.sync(config, dml);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+3
-12
@@ -6,9 +6,7 @@ import org.slf4j.LoggerFactory;
|
||||
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
|
||||
import com.alibaba.otter.canal.client.adapter.support.SPI;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
* 外部适配器示例
|
||||
@@ -22,20 +20,13 @@ public class LoggerAdapterExample implements CanalOuterAdapter {
|
||||
private Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||
|
||||
@Override
|
||||
public void writeOut(Message message) {
|
||||
// 直接输出日志信息
|
||||
MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
|
||||
public void init(CanalOuterAdapterConfiguration configuration) {
|
||||
|
||||
@Override
|
||||
public void accept(Dml dml) {
|
||||
logger.info(dml.toString());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(CanalOuterAdapterConfiguration configuration) {
|
||||
|
||||
public void writeOut(Dml dml) {
|
||||
logger.info(dml.toString());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -74,7 +74,7 @@ else
|
||||
fi
|
||||
|
||||
JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
|
||||
CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
|
||||
CANAL_OPTS="-DappName=otter-canal-client -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
|
||||
|
||||
if [ -e $canal_conf -a -e $logback_configurationFile ]
|
||||
then
|
||||
|
||||
@@ -45,7 +45,7 @@ fi
|
||||
|
||||
pid=`cat $pidfile`
|
||||
if [ "$pid" == "" ] ; then
|
||||
pid=`get_pid "appName=otter-canal"`
|
||||
pid=`get_pid "appName=otter-canal-client"`
|
||||
fi
|
||||
|
||||
echo -e "`hostname`: stopping canal $pid ... "
|
||||
@@ -54,7 +54,7 @@ kill $pid
|
||||
LOOPS=0
|
||||
while (true);
|
||||
do
|
||||
gpid=`get_pid "appName=otter-canal" "$pid"`
|
||||
gpid=`get_pid "appName=otter-canal-client" "$pid"`
|
||||
if [ "$gpid" == "" ] ; then
|
||||
echo "Oook! cost:$LOOPS"
|
||||
`rm $pidfile`
|
||||
|
||||
+56
-4
@@ -7,10 +7,13 @@ import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.support.Dml;
|
||||
import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
@@ -47,9 +50,16 @@ public abstract class AbstractCanalAdapterWorker {
|
||||
public Boolean call() {
|
||||
try {
|
||||
// 组内适配器穿行运行，尽量不要配置组内适配器
|
||||
for (CanalOuterAdapter c : adapters) {
|
||||
for (final CanalOuterAdapter c : adapters) {
|
||||
long begin = System.currentTimeMillis();
|
||||
c.writeOut(message);
|
||||
MessageUtil.parse4Dml(message, new MessageUtil.Consumer<Dml>() {
|
||||
|
||||
@Override
|
||||
public void accept(Dml dml) {
|
||||
c.writeOut(dml);
|
||||
}
|
||||
});
|
||||
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("{} elapsed time: {}",
|
||||
c.getClass().getName(),
|
||||
@@ -71,7 +81,49 @@ public abstract class AbstractCanalAdapterWorker {
|
||||
logger.error("Outer adapter write failed");
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeOut(final FlatMessage flatMessage) {
|
||||
List<Future<Boolean>> futures = new ArrayList<>();
|
||||
// 组间适配器并行运行
|
||||
for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
|
||||
final List<CanalOuterAdapter> adapters = outerAdapters;
|
||||
futures.add(groupInnerExecutorService.submit(new Callable<Boolean>() {
|
||||
|
||||
@Override
|
||||
public Boolean call() {
|
||||
try {
|
||||
// 组内适配器穿行运行，尽量不要配置组内适配器
|
||||
for (CanalOuterAdapter c : adapters) {
|
||||
long begin = System.currentTimeMillis();
|
||||
Dml dml = MessageUtil.flatMessage2Dml(flatMessage);
|
||||
c.writeOut(dml);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("{} elapsed time: {}",
|
||||
c.getClass().getName(),
|
||||
(System.currentTimeMillis() - begin));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// 等待所有适配器写入完成
|
||||
// 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
|
||||
for (Future<Boolean> f : futures) {
|
||||
try {
|
||||
if (!f.get()) {
|
||||
logger.error("Outer adapter write failed");
|
||||
}
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+69
-32
@@ -12,6 +12,7 @@ import org.apache.kafka.common.errors.WakeupException;
|
||||
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
|
||||
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
@@ -24,16 +25,19 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
|
||||
|
||||
private KafkaCanalConnector connector;
|
||||
|
||||
private String topic;
|
||||
private String topic;
|
||||
|
||||
public CanalAdapterKafkaWorker(String zkServers, String bootstrapServers, String topic, String groupId,
|
||||
List<List<CanalOuterAdapter>> canalOuterAdapters) {
|
||||
private boolean flatMessage;
|
||||
|
||||
public CanalAdapterKafkaWorker(String bootstrapServers, String topic, String groupId,
|
||||
List<List<CanalOuterAdapter>> canalOuterAdapters, boolean flatMessage){
|
||||
this.canalOuterAdapters = canalOuterAdapters;
|
||||
this.groupInnerExecutorService = Executors.newFixedThreadPool(canalOuterAdapters.size());
|
||||
this.topic = topic;
|
||||
this.canalDestination = topic;
|
||||
connector = KafkaCanalConnectors.newKafkaConnector(zkServers, bootstrapServers, topic, null, groupId);
|
||||
// connector.setSessionTimeout(5L, TimeUnit.MINUTES);
|
||||
this.flatMessage = flatMessage;
|
||||
connector = KafkaCanalConnectors.newKafkaConnector(bootstrapServers, topic, null, groupId, flatMessage);
|
||||
// connector.setSessionTimeout(1L, TimeUnit.MINUTES);
|
||||
|
||||
// super.initSwitcher(topic);
|
||||
}
|
||||
@@ -60,11 +64,29 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
|
||||
if (!running) {
|
||||
return;
|
||||
}
|
||||
|
||||
connector.stopRunning();
|
||||
running = false;
|
||||
logger.info("Stop topic {} out adapters begin", this.topic);
|
||||
stopOutAdapters();
|
||||
logger.info("Stop topic {} out adapters end", this.topic);
|
||||
|
||||
// if (switcher != null && !switcher.state()) {
|
||||
// switcher.set(true);
|
||||
// }
|
||||
|
||||
if (thread != null) {
|
||||
try {
|
||||
thread.join();
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
groupInnerExecutorService.shutdown();
|
||||
logger.info("topic {} connectors' worker thread dead!", this.topic);
|
||||
for (List<CanalOuterAdapter> outerAdapters : canalOuterAdapters) {
|
||||
for (CanalOuterAdapter adapter : outerAdapters) {
|
||||
adapter.destroy();
|
||||
}
|
||||
}
|
||||
logger.info("topic {} all connectors destroyed!", this.topic);
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
@@ -79,39 +101,54 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
|
||||
try {
|
||||
logger.info("=============> Start to connect topic: {} <=============", this.topic);
|
||||
connector.connect();
|
||||
logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
|
||||
logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
|
||||
connector.subscribe();
|
||||
logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
|
||||
logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
|
||||
while (running) {
|
||||
try {
|
||||
// switcher.get(); //等待开关开启
|
||||
|
||||
final Message message = connector.getWithoutAck();
|
||||
List<?> messages;
|
||||
if (!flatMessage) {
|
||||
messages = connector.getWithoutAck();
|
||||
} else {
|
||||
messages = connector.getFlatMessageWithoutAck(100L, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
if (messages != null) {
|
||||
for (final Object message : messages) {
|
||||
executing.set(true);
|
||||
if (message != null) {
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
executing.set(true);
|
||||
if (message != null) {
|
||||
executor.submit(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
if (message instanceof FlatMessage) {
|
||||
writeOut((FlatMessage) message);
|
||||
} else {
|
||||
writeOut((Message) message);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
executing.compareAndSet(true, false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
writeOut(message, topic);
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
} finally {
|
||||
executing.compareAndSet(true, false);
|
||||
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
|
||||
long currentTS = System.currentTimeMillis();
|
||||
while (executing.get()) {
|
||||
// 大于10秒未消费完ack一次keep alive
|
||||
if (System.currentTimeMillis() - currentTS > 10000) {
|
||||
connector.ack();
|
||||
currentTS = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// 间隔一段时间ack一次, 防止因超时未响应切换到另外台客户端
|
||||
while (executing.get()) {
|
||||
connector.ack();
|
||||
Thread.sleep(500);
|
||||
}
|
||||
} else {
|
||||
connector.ack();
|
||||
}
|
||||
connector.ack();
|
||||
} catch (CommitFailedException e) {
|
||||
logger.warn(e.getMessage());
|
||||
} catch (Exception e) {
|
||||
@@ -131,7 +168,7 @@ public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
|
||||
} catch (WakeupException e) {
|
||||
// No-op. Continue process
|
||||
}
|
||||
connector.disconnnect();
|
||||
connector.disconnect();
|
||||
logger.info("=============> Disconnect topic: {} <=============", this.topic);
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-7
@@ -4,10 +4,6 @@ import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
|
||||
import com.alibaba.otter.canal.client.adapter.support.CanalOuterAdapterConfiguration;
|
||||
import com.alibaba.otter.canal.client.adapter.support.ExtensionLoader;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.util.ArrayList;
|
||||
@@ -16,6 +12,9 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* MQ外部适配器的加载器
|
||||
@@ -94,7 +93,6 @@ public class CanalAdapterLoader {
|
||||
loadConnector(config, canalOuterAdapters);
|
||||
}
|
||||
canalOuterAdapterGroups.add(canalOuterAdapters);
|
||||
|
||||
if (StringUtils.isBlank(topic.getMqMode()) || "rocketmq".equalsIgnoreCase(topic.getMqMode())) {
|
||||
CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(
|
||||
canalClientConfig.getBootstrapServers(),
|
||||
@@ -104,16 +102,17 @@ public class CanalAdapterLoader {
|
||||
canalMQWorker.put(topic.getTopic() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
|
||||
rocketMQWorker.start();
|
||||
} else if ("kafka".equalsIgnoreCase(topic.getMqMode())) {
|
||||
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(zkHosts,
|
||||
CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(
|
||||
canalClientConfig.getBootstrapServers(),
|
||||
topic.getTopic(),
|
||||
group.getGroupId(),
|
||||
canalOuterAdapterGroups);
|
||||
canalOuterAdapterGroups,canalClientConfig.getFlatMessage());
|
||||
canalMQWorker.put(topic.getTopic() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
|
||||
canalKafkaWorker.start();
|
||||
}
|
||||
logger.info("Start adapter for canal-client rocketmq topic: {} succeed",
|
||||
topic.getTopic() + "-" + group.getGroupId());
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+6
-7
@@ -1,16 +1,15 @@
|
||||
package com.alibaba.otter.canal.client.adapter.loader;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.CanalConnectors;
|
||||
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import com.alibaba.otter.canal.client.adapter.loader.AbstractCanalAdapterWorker;
|
||||
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
import java.net.SocketAddress;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.CanalConnectors;
|
||||
import com.alibaba.otter.canal.client.adapter.CanalOuterAdapter;
|
||||
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
* 原生canal-server对应的client适配器工作线程
|
||||
*
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
canalServerHost: 127.0.0.1:11111
|
||||
#canalServerHost: 127.0.0.1:11111
|
||||
#zookeeperHosts: slave1:2181
|
||||
#bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers
|
||||
bootstrapServers: slave1:6667,slave2:6667 #or rocketmq nameservers
|
||||
flatMessage: true
|
||||
|
||||
#canalInstances:
|
||||
#- instance: example
|
||||
|
||||
+262
-112
@@ -1,115 +1,265 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.1.1-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal client module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- 客户端要使用请单独引入kafka依赖 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-client</artifactId>
|
||||
<version>4.3.0</version>
|
||||
</dependency>
|
||||
<!-- junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal</artifactId>
|
||||
<version>1.1.1-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.client</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<name>canal client module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.protocol</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
</profile>
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
</dependency>
|
||||
|
||||
<profile>
|
||||
<id>javadoc</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<aggregate>true</aggregate>
|
||||
<show>public</show>
|
||||
<nohelp>true</nohelp>
|
||||
<header>${project.artifactId}-${project.version}</header>
|
||||
<footer>${project.artifactId}-${project.version}</footer>
|
||||
<doctitle>${project.artifactId}-${project.version}</doctitle>
|
||||
<links>
|
||||
<link>https://github.com/alibaba/canal</link>
|
||||
</links>
|
||||
<outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}
|
||||
</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-scm-publish-plugin</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>publish-scm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
|
||||
<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}
|
||||
</checkinComment>
|
||||
<content>${project.build.directory}/apidocs</content>
|
||||
<skipDeletedFiles>true</skipDeletedFiles>
|
||||
<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
|
||||
<scmBranch>gh-pages</scmBranch>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
|
||||
<dependency>
|
||||
<groupId>io.netty</groupId>
|
||||
<artifactId>netty-all</artifactId>
|
||||
</dependency>
|
||||
<!-- zk -->
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.101tec</groupId>
|
||||
<artifactId>zkclient</artifactId>
|
||||
</dependency>
|
||||
<!-- external -->
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
<!-- log -->
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
<artifactId>logback-classic</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>jcl-over-slf4j</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-aop</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-context</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-orm</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<version>${spring_version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.rocketmq</groupId>
|
||||
<artifactId>rocketmq-client</artifactId>
|
||||
<version>4.3.0</version>
|
||||
</dependency>
|
||||
<!-- 客户端要使用请单独引入kafka依赖 -->
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<version>1.1.1</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- junit -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-shade-plugin</artifactId>
|
||||
<version>2.4.1</version>
|
||||
<configuration>
|
||||
<createDependencyReducedPom>false</createDependencyReducedPom>
|
||||
<artifactSet>
|
||||
<excludes>
|
||||
<exclude>org.apache.zookeeper:zookeeper</exclude>
|
||||
<exclude>com.101tec:zkclient</exclude>
|
||||
<exclude>commons-io:commons-io</exclude>
|
||||
<exclude>commons-lang:commons-lang</exclude>
|
||||
<exclude>commons-codec:commons-codec</exclude>
|
||||
<exclude>com.alibaba:fastjson</exclude>
|
||||
<exclude>ch.qos.logback:logback-core</exclude>
|
||||
<exclude>ch.qos.logback:logback-classic</exclude>
|
||||
<exclude>org.slf4j:jcl-over-slf4j</exclude>
|
||||
<exclude>org.slf4j:slf4j-api</exclude>
|
||||
<exclude>org.springframework:*</exclude>
|
||||
<exclude>org.jboss.netty:netty</exclude>
|
||||
<exclude>commons-logging:commons-logging</exclude>
|
||||
<exclude>aopalliance:aopalliance</exclude>
|
||||
<exclude>com.google.protobuf:protobuf-java</exclude>
|
||||
<exclude>io.netty:*</exclude>
|
||||
<exclude>junit:junit</exclude>
|
||||
</excludes>
|
||||
</artifactSet>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>shade</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<relocations>
|
||||
<relocation>
|
||||
<pattern>com.google.guava</pattern>
|
||||
<shadedPattern>com.alibaba.google.guava</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>org.joda</pattern>
|
||||
<shadedPattern>com.alibaba.google.joda</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.common</pattern>
|
||||
<shadedPattern>com.alibaba.google.common</shadedPattern>
|
||||
</relocation>
|
||||
<relocation>
|
||||
<pattern>com.google.thirdparty</pattern>
|
||||
<shadedPattern>com.alibaba.google.thirdparty</shadedPattern>
|
||||
</relocation>
|
||||
</relocations>
|
||||
<transformers>
|
||||
<transformer
|
||||
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
|
||||
</transformers>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dev</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>!javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>javadoc</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>env</name>
|
||||
<value>javadoc</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<aggregate>true</aggregate>
|
||||
<show>public</show>
|
||||
<nohelp>true</nohelp>
|
||||
<header>${project.artifactId}-${project.version}</header>
|
||||
<footer>${project.artifactId}-${project.version}</footer>
|
||||
<doctitle>${project.artifactId}-${project.version}</doctitle>
|
||||
<links>
|
||||
<link>https://github.com/alibaba/canal</link>
|
||||
</links>
|
||||
<outputDirectory>${project.build.directory}/apidocs/apidocs/${project.version}</outputDirectory>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-scm-publish-plugin</artifactId>
|
||||
<version>1.0-beta-2</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>publish-scm</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
<configuration>
|
||||
<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
|
||||
<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}</checkinComment>
|
||||
<content>${project.build.directory}/apidocs</content>
|
||||
<skipDeletedFiles>true</skipDeletedFiles>
|
||||
<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
|
||||
<scmBranch>gh-pages</scmBranch>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
+5
-5
@@ -8,8 +8,8 @@ import com.google.protobuf.ByteString;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class DeserializerUtil {
|
||||
private static Logger logger = LoggerFactory.getLogger(DeserializerUtil.class);
|
||||
public class CanalMessageDeserializer {
|
||||
private static Logger logger = LoggerFactory.getLogger(CanalMessageDeserializer.class);
|
||||
|
||||
public static Message deserializer(byte[] data) {
|
||||
try {
|
||||
@@ -19,9 +19,9 @@ public class DeserializerUtil {
|
||||
CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
|
||||
switch (p.getType()) {
|
||||
case MESSAGES: {
|
||||
if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
|
||||
throw new CanalClientException("compression is not supported in this connector");
|
||||
}
|
||||
// if (!p.getCompression().equals(CanalPacket.Compression.NONE)) {
|
||||
// throw new CanalClientException("compression is not supported in this connector");
|
||||
// }
|
||||
|
||||
CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
|
||||
Message result = new Message(messages.getBatchId());
|
||||
@@ -159,7 +159,7 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
}
|
||||
//
|
||||
Handshake handshake = Handshake.parseFrom(p.getBody());
|
||||
supportedCompressions.addAll(handshake.getSupportedCompressionsList());
|
||||
supportedCompressions.add(handshake.getSupportedCompressions());
|
||||
//
|
||||
ClientAuth ca = ClientAuth.newBuilder()
|
||||
.setUsername(username != null ? username : "")
|
||||
@@ -180,8 +180,8 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
|
||||
Ack ackBody = Ack.parseFrom(ack.getBody());
|
||||
if (ackBody.getErrorCode() > 0) {
|
||||
throw new CanalClientException("something goes wrong when doing authentication: "
|
||||
+ ackBody.getErrorMessage());
|
||||
throw new CanalClientException(
|
||||
"something goes wrong when doing authentication: " + ackBody.getErrorMessage());
|
||||
}
|
||||
|
||||
connected = true;
|
||||
@@ -323,7 +323,8 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
Packet p = Packet.parseFrom(data);
|
||||
switch (p.getType()) {
|
||||
case MESSAGES: {
|
||||
if (!p.getCompression().equals(Compression.NONE)) {
|
||||
if (!p.getCompression().equals(Compression.NONE)
|
||||
&& !p.getCompression().equals(Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
|
||||
throw new CanalClientException("compression is not supported in this connector");
|
||||
}
|
||||
|
||||
@@ -360,11 +361,8 @@ public class SimpleCanalConnector implements CanalConnector {
|
||||
.setBatchId(batchId)
|
||||
.build();
|
||||
try {
|
||||
writeWithHeader(Packet.newBuilder()
|
||||
.setType(PacketType.CLIENTACK)
|
||||
.setBody(ca.toByteString())
|
||||
.build()
|
||||
.toByteArray());
|
||||
writeWithHeader(
|
||||
Packet.newBuilder().setType(PacketType.CLIENTACK).setBody(ca.toByteString()).build().toByteArray());
|
||||
} catch (IOException e) {
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
|
||||
+85
-77
@@ -1,19 +1,20 @@
|
||||
package com.alibaba.otter.canal.client.kafka;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
||||
import org.apache.kafka.clients.consumer.KafkaConsumer;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.kafka.common.serialization.StringDeserializer;
|
||||
|
||||
import com.alibaba.otter.canal.client.kafka.running.ClientRunningData;
|
||||
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
|
||||
/**
|
||||
* canal kafka 数据操作客户端
|
||||
@@ -24,18 +25,19 @@ import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
public class KafkaCanalConnector {
|
||||
|
||||
private KafkaConsumer<String, Message> kafkaConsumer;
|
||||
private KafkaConsumer<String, String> kafkaConsumer2; // 用于扁平message的数据消费
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
private Properties properties;
|
||||
// private ClientRunningMonitor runningMonitor; // 运行控制
|
||||
// private BooleanMutex mutex = new BooleanMutex(false);
|
||||
private ZkClientx zkClientx;
|
||||
private volatile boolean connected = false;
|
||||
private volatile boolean running = false;
|
||||
private boolean flatMessage;
|
||||
|
||||
public KafkaCanalConnector(String zkServers, String servers, String topic, Integer partition, String groupId){
|
||||
public KafkaCanalConnector(String servers, String topic, Integer partition, String groupId,
|
||||
boolean flatMessage){
|
||||
this.topic = topic;
|
||||
this.partition = partition;
|
||||
this.flatMessage = flatMessage;
|
||||
|
||||
properties = new Properties();
|
||||
properties.put("bootstrap.servers", servers);
|
||||
@@ -45,32 +47,13 @@ public class KafkaCanalConnector {
|
||||
properties.put("auto.offset.reset", "latest"); // 如果没有offset则从最后的offset开始读
|
||||
properties.put("request.timeout.ms", "40000"); // 必须大于session.timeout.ms的设置
|
||||
properties.put("session.timeout.ms", "30000"); // 默认为30秒
|
||||
properties.put("max.poll.records", "1"); // 所以一次只取一条数据
|
||||
properties.put("max.poll.records", "100");
|
||||
properties.put("key.deserializer", StringDeserializer.class.getName());
|
||||
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
||||
|
||||
if (zkServers != null) {
|
||||
zkClientx = new ZkClientx(zkServers);
|
||||
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setGroupId(groupId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
|
||||
// runningMonitor = new ClientRunningMonitor();
|
||||
// runningMonitor.setTopic(topic);
|
||||
// runningMonitor.setZkClient(zkClientx);
|
||||
// runningMonitor.setClientData(clientData);
|
||||
// runningMonitor.setListener(new ClientRunningListener() {
|
||||
// public void processActiveEnter() {
|
||||
// mutex.set(true);
|
||||
// }
|
||||
//
|
||||
// public void processActiveExit() {
|
||||
// mutex.set(false);
|
||||
// }
|
||||
// });
|
||||
if (!flatMessage) {
|
||||
properties.put("value.deserializer", MessageDeserializer.class.getName());
|
||||
} else {
|
||||
properties.put("value.deserializer", StringDeserializer.class.getName());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -93,57 +76,36 @@ public class KafkaCanalConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
// if (runningMonitor != null) {
|
||||
// if (!runningMonitor.isStart()) {
|
||||
// runningMonitor.start();
|
||||
// }
|
||||
// }
|
||||
|
||||
connected = true;
|
||||
|
||||
if (kafkaConsumer == null) {
|
||||
if (kafkaConsumer == null && !flatMessage) {
|
||||
kafkaConsumer = new KafkaConsumer<String, Message>(properties);
|
||||
}
|
||||
if (kafkaConsumer2 == null && flatMessage) {
|
||||
kafkaConsumer2 = new KafkaConsumer<String, String>(properties);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭链接
|
||||
*/
|
||||
public void disconnnect() {
|
||||
kafkaConsumer.close();
|
||||
public void disconnect() {
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.close();
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.close();
|
||||
}
|
||||
|
||||
connected = false;
|
||||
// if (runningMonitor.isStart()) {
|
||||
// runningMonitor.stop();
|
||||
// }
|
||||
}
|
||||
|
||||
private void waitClientRunning() {
|
||||
try {
|
||||
if (zkClientx != null) {
|
||||
if (!connected) {// 未调用connect
|
||||
throw new CanalClientException("should connect first");
|
||||
}
|
||||
|
||||
running = true;
|
||||
// mutex.get();// 阻塞等待
|
||||
} else {
|
||||
// 单机模式直接设置为running
|
||||
running = true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new CanalClientException(e);
|
||||
}
|
||||
running = true;
|
||||
}
|
||||
|
||||
public boolean checkValid() {
|
||||
if (zkClientx != null) {
|
||||
// return mutex.state();
|
||||
return true;
|
||||
} else {
|
||||
return true;// 默认都放过
|
||||
}
|
||||
return true;// 默认都放过
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -156,10 +118,20 @@ public class KafkaCanalConnector {
|
||||
}
|
||||
|
||||
if (partition == null) {
|
||||
kafkaConsumer.subscribe(Collections.singletonList(topic));
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.subscribe(Collections.singletonList(topic));
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.subscribe(Collections.singletonList(topic));
|
||||
}
|
||||
} else {
|
||||
TopicPartition topicPartition = new TopicPartition(topic, partition);
|
||||
kafkaConsumer.assign(Collections.singletonList(topicPartition));
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.assign(Collections.singletonList(topicPartition));
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.assign(Collections.singletonList(topicPartition));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,7 +144,12 @@ public class KafkaCanalConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
kafkaConsumer.unsubscribe();
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.unsubscribe();
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.unsubscribe();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -180,22 +157,22 @@ public class KafkaCanalConnector {
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Message get() {
|
||||
public List<Message> get() {
|
||||
return get(100L, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
public Message get(Long timeout, TimeUnit unit) {
|
||||
public List<Message> get(Long timeout, TimeUnit unit) {
|
||||
waitClientRunning();
|
||||
if (!running) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Message message = getWithoutAck(timeout, unit);
|
||||
List<Message> messages = getWithoutAck(timeout, unit);
|
||||
this.ack();
|
||||
return message;
|
||||
return messages;
|
||||
}
|
||||
|
||||
public Message getWithoutAck() {
|
||||
public List<Message> getWithoutAck() {
|
||||
return getWithoutAck(100L, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@@ -204,7 +181,7 @@ public class KafkaCanalConnector {
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Message getWithoutAck(Long timeout, TimeUnit unit) {
|
||||
public List<Message> getWithoutAck(Long timeout, TimeUnit unit) {
|
||||
waitClientRunning();
|
||||
if (!running) {
|
||||
return null;
|
||||
@@ -213,7 +190,33 @@ public class KafkaCanalConnector {
|
||||
ConsumerRecords<String, Message> records = kafkaConsumer.poll(unit.toMillis(timeout)); // 基于配置，最多只能poll到一条数据
|
||||
|
||||
if (!records.isEmpty()) {
|
||||
return records.iterator().next().value();
|
||||
// return records.iterator().next().value();
|
||||
List<Message> messages = new ArrayList<>();
|
||||
for (ConsumerRecord<String, Message> record : records) {
|
||||
messages.add(record.value());
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public List<FlatMessage> getFlatMessageWithoutAck(Long timeout, TimeUnit unit) {
|
||||
waitClientRunning();
|
||||
if (!running) {
|
||||
return null;
|
||||
}
|
||||
|
||||
ConsumerRecords<String, String> records = kafkaConsumer2.poll(unit.toMillis(timeout));
|
||||
|
||||
if (!records.isEmpty()) {
|
||||
List<FlatMessage> flatMessages = new ArrayList<>();
|
||||
for (ConsumerRecord<String, String> record : records) {
|
||||
String flatMessageJson = record.value();
|
||||
FlatMessage flatMessage = JSON.parseObject(flatMessageJson, FlatMessage.class);
|
||||
flatMessages.add(flatMessage);
|
||||
}
|
||||
|
||||
return flatMessages;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -227,7 +230,12 @@ public class KafkaCanalConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
kafkaConsumer.commitSync();
|
||||
if (kafkaConsumer != null) {
|
||||
kafkaConsumer.commitSync();
|
||||
}
|
||||
if (kafkaConsumer2 != null) {
|
||||
kafkaConsumer2.commitAsync();
|
||||
}
|
||||
}
|
||||
|
||||
public void stopRunning() {
|
||||
|
||||
+8
-19
@@ -17,8 +17,9 @@ public class KafkaCanalConnectors {
|
||||
* @param groupId
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition, String groupId) {
|
||||
return new KafkaCanalConnector(null, servers, topic, partition, groupId);
|
||||
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, Integer partition,
|
||||
String groupId) {
|
||||
return new KafkaCanalConnector(servers, topic, partition, groupId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -30,33 +31,21 @@ public class KafkaCanalConnectors {
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String servers, String topic, String groupId) {
|
||||
return new KafkaCanalConnector(null, servers, topic, null, groupId);
|
||||
return new KafkaCanalConnector(servers, topic, null, groupId, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建kafka客户端链接
|
||||
*
|
||||
* @param zkServers
|
||||
* @param servers
|
||||
* @param topic
|
||||
* @param partition
|
||||
* @param groupId
|
||||
* @param flatMessage
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, Integer partition, String groupId) {
|
||||
return new KafkaCanalConnector(zkServers, servers, topic, partition, groupId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建kafka客户端链接
|
||||
*
|
||||
* @param zkServers
|
||||
* @param servers
|
||||
* @param topic
|
||||
* @param groupId
|
||||
* @return
|
||||
*/
|
||||
public static KafkaCanalConnector newKafkaConnector(String zkServers, String servers, String topic, String groupId) {
|
||||
return new KafkaCanalConnector(zkServers, servers, topic, null, groupId);
|
||||
public static KafkaCanalConnector newKafkaConnector( String servers, String topic,
|
||||
Integer partition, String groupId,boolean flatMessage) {
|
||||
return new KafkaCanalConnector(servers, topic, partition, groupId, flatMessage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.kafka;
|
||||
|
||||
import com.alibaba.otter.canal.client.DeserializerUtil;
|
||||
import com.alibaba.otter.canal.client.CanalMessageDeserializer;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import java.util.Map;
|
||||
import org.apache.kafka.common.serialization.Deserializer;
|
||||
@@ -23,7 +23,7 @@ public class MessageDeserializer implements Deserializer<Message> {
|
||||
|
||||
@Override
|
||||
public Message deserialize(String topic1, byte[] data) {
|
||||
return DeserializerUtil.deserializer(data);
|
||||
return CanalMessageDeserializer.deserializer(data);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
package com.alibaba.otter.canal.client.rocketmq;
|
||||
|
||||
import com.alibaba.otter.canal.client.CanalConnector;
|
||||
import com.alibaba.otter.canal.client.DeserializerUtil;
|
||||
import com.alibaba.otter.canal.client.CanalMessageDeserializer;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||
import java.util.List;
|
||||
@@ -94,7 +94,7 @@ public class RocketMQCanalConnector implements CanalConnector {
|
||||
BlockingQueue<Message> messageList = new LinkedBlockingQueue<>();
|
||||
for (MessageExt messageExt : messageExts) {
|
||||
byte[] data = messageExt.getBody();
|
||||
Message message = DeserializerUtil.deserializer(data);
|
||||
Message message = CanalMessageDeserializer.deserializer(data);
|
||||
try {
|
||||
messageList.put(message);
|
||||
} catch (InterruptedException ex) {
|
||||
|
||||
+94
-94
@@ -20,117 +20,117 @@ import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||
|
||||
public class ClientRunningTest extends AbstractZkTest {
|
||||
|
||||
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||
private short clientId = 1001;
|
||||
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||
private short clientId = 1001;
|
||||
|
||||
@Before
|
||||
public void setUp() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
@Before
|
||||
public void setUp() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
|
||||
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
|
||||
}
|
||||
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
}
|
||||
@After
|
||||
public void tearDown() {
|
||||
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||
zkclientx.deleteRecursive(path);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOneServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
|
||||
runningMonitor.start();
|
||||
sleep(2000L);
|
||||
runningMonitor.stop();
|
||||
sleep(2000L);
|
||||
@Test
|
||||
public void testOneServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
|
||||
runningMonitor.start();
|
||||
sleep(2000L);
|
||||
runningMonitor.stop();
|
||||
sleep(2000L);
|
||||
|
||||
if (countLatch.getCount() != 0) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
if (countLatch.getCount() != 0) {
|
||||
Assert.fail();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
|
||||
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
|
||||
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
|
||||
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||
executor.submit(new Runnable() {
|
||||
@Test
|
||||
public void testMultiServer() {
|
||||
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
|
||||
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
|
||||
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
|
||||
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor1.isStart()) {
|
||||
runningMonitor1.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor1.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor1.isStart()) {
|
||||
runningMonitor1.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor1.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
executor.submit(new Runnable() {
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor2.isStart()) {
|
||||
runningMonitor2.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor2.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor2.isStart()) {
|
||||
runningMonitor2.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor2.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
executor.submit(new Runnable() {
|
||||
executor.submit(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor3.isStart()) {
|
||||
runningMonitor3.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor3.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
public void run() {
|
||||
for (int i = 0; i < 10; i++) {
|
||||
if (!runningMonitor3.isStart()) {
|
||||
runningMonitor3.start();
|
||||
}
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
runningMonitor3.stop();
|
||||
sleep(2000L + RandomUtils.nextInt(500));
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
});
|
||||
|
||||
sleep(30000L);
|
||||
}
|
||||
sleep(30000L);
|
||||
}
|
||||
|
||||
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
|
||||
final int port) {
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setClientId(clientId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
|
||||
final int port) {
|
||||
ClientRunningData clientData = new ClientRunningData();
|
||||
clientData.setClientId(clientId);
|
||||
clientData.setAddress(AddressUtils.getHostIp());
|
||||
|
||||
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
|
||||
runningMonitor.setDestination(destination);
|
||||
runningMonitor.setZkClient(zkclientx);
|
||||
runningMonitor.setClientData(clientData);
|
||||
runningMonitor.setListener(new ClientRunningListener() {
|
||||
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
|
||||
runningMonitor.setDestination(destination);
|
||||
runningMonitor.setZkClient(zkclientx);
|
||||
runningMonitor.setClientData(clientData);
|
||||
runningMonitor.setListener(new ClientRunningListener() {
|
||||
|
||||
public InetSocketAddress processActiveEnter() {
|
||||
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
|
||||
countLatch.countDown();
|
||||
return new InetSocketAddress(AddressUtils.getHostIp(), port);
|
||||
}
|
||||
public InetSocketAddress processActiveEnter() {
|
||||
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
|
||||
countLatch.countDown();
|
||||
return new InetSocketAddress(AddressUtils.getHostIp(), port);
|
||||
}
|
||||
|
||||
public void processActiveExit() {
|
||||
countLatch.countDown();
|
||||
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
|
||||
}
|
||||
public void processActiveExit() {
|
||||
countLatch.countDown();
|
||||
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
|
||||
}
|
||||
|
||||
});
|
||||
runningMonitor.setDelayTime(1);
|
||||
return runningMonitor;
|
||||
}
|
||||
}
|
||||
});
|
||||
runningMonitor.setDelayTime(1);
|
||||
return runningMonitor;
|
||||
}
|
||||
}
|
||||
+34
-29
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.running.kafka;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.kafka.common.errors.WakeupException;
|
||||
@@ -19,32 +20,33 @@ import com.alibaba.otter.canal.protocol.Message;
|
||||
*/
|
||||
public class CanalKafkaClientExample {
|
||||
|
||||
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
|
||||
protected final static Logger logger = LoggerFactory.getLogger(CanalKafkaClientExample.class);
|
||||
|
||||
private KafkaCanalConnector connector;
|
||||
private KafkaCanalConnector connector;
|
||||
|
||||
private static volatile boolean running = false;
|
||||
private static volatile boolean running = false;
|
||||
|
||||
private Thread thread = null;
|
||||
private Thread thread = null;
|
||||
|
||||
private Thread.UncaughtExceptionHandler handler = new Thread.UncaughtExceptionHandler() {
|
||||
|
||||
public void uncaughtException(Thread t, Throwable e) {
|
||||
logger.error("parse events has an error", e);
|
||||
}
|
||||
};
|
||||
public void uncaughtException(Thread t, Throwable e) {
|
||||
logger.error("parse events has an error", e);
|
||||
}
|
||||
};
|
||||
|
||||
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId) {
|
||||
connector = KafkaCanalConnectors.newKafkaConnector(zkServers, servers, topic, partition, groupId);
|
||||
public CanalKafkaClientExample(String zkServers, String servers, String topic, Integer partition, String groupId){
|
||||
connector = KafkaCanalConnectors.newKafkaConnector(servers, topic, partition, groupId, false);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(AbstractKafkaTest.zkServers,
|
||||
AbstractKafkaTest.servers,
|
||||
AbstractKafkaTest.topic,
|
||||
AbstractKafkaTest.partition,
|
||||
AbstractKafkaTest.groupId);
|
||||
final CanalKafkaClientExample kafkaCanalClientExample = new CanalKafkaClientExample(
|
||||
AbstractKafkaTest.zkServers,
|
||||
AbstractKafkaTest.servers,
|
||||
AbstractKafkaTest.topic,
|
||||
AbstractKafkaTest.partition,
|
||||
AbstractKafkaTest.groupId);
|
||||
logger.info("## start the kafka consumer: {}-{}", AbstractKafkaTest.topic, AbstractKafkaTest.groupId);
|
||||
kafkaCanalClientExample.start();
|
||||
logger.info("## the canal kafka consumer is running now ......");
|
||||
@@ -107,21 +109,24 @@ public class CanalKafkaClientExample {
|
||||
connector.subscribe();
|
||||
while (running) {
|
||||
try {
|
||||
Message message = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
|
||||
if (message == null) {
|
||||
List<Message> messages = connector.getWithoutAck(1L, TimeUnit.SECONDS); // 获取message
|
||||
|
||||
if (messages == null) {
|
||||
continue;
|
||||
}
|
||||
long batchId = message.getId();
|
||||
int size = message.getEntries().size();
|
||||
if (batchId == -1 || size == 0) {
|
||||
// try {
|
||||
// Thread.sleep(1000);
|
||||
// } catch (InterruptedException e) {
|
||||
// }
|
||||
} else {
|
||||
// printSummary(message, batchId, size);
|
||||
// printEntry(message.getEntries());
|
||||
logger.info(message.toString());
|
||||
for (Message message : messages) {
|
||||
long batchId = message.getId();
|
||||
int size = message.getEntries().size();
|
||||
if (batchId == -1 || size == 0) {
|
||||
// try {
|
||||
// Thread.sleep(1000);
|
||||
// } catch (InterruptedException e) {
|
||||
// }
|
||||
} else {
|
||||
// printSummary(message, batchId, size);
|
||||
// printEntry(message.getEntries());
|
||||
logger.info(message.toString());
|
||||
}
|
||||
}
|
||||
|
||||
connector.ack(); // 提交确认
|
||||
@@ -139,6 +144,6 @@ public class CanalKafkaClientExample {
|
||||
} catch (WakeupException e) {
|
||||
// No-op. Continue process
|
||||
}
|
||||
connector.disconnnect();
|
||||
connector.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
+5
-4
@@ -1,5 +1,6 @@
|
||||
package com.alibaba.otter.canal.client.running.kafka;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -39,9 +40,9 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
|
||||
connector.subscribe();
|
||||
while (running) {
|
||||
try {
|
||||
Message message = connector.getWithoutAck(3L, TimeUnit.SECONDS);
|
||||
if (message != null) {
|
||||
System.out.println(message);
|
||||
List<Message> messages = connector.getWithoutAck(3L, TimeUnit.SECONDS);
|
||||
if (messages != null) {
|
||||
System.out.println(messages);
|
||||
}
|
||||
connector.ack();
|
||||
} catch (WakeupException e) {
|
||||
@@ -49,7 +50,7 @@ public class KafkaClientRunningTest extends AbstractKafkaTest {
|
||||
}
|
||||
}
|
||||
connector.unsubscribe();
|
||||
connector.disconnnect();
|
||||
connector.disconnect();
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -11,6 +11,11 @@
|
||||
<packaging>jar</packaging>
|
||||
<name>canal dbsync module for otter ${project.version}</name>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.otter</groupId>
|
||||
<artifactId>canal.parse.driver</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<!-- log -->
|
||||
<dependency>
|
||||
<groupId>ch.qos.logback</groupId>
|
||||
|
||||
@@ -3,7 +3,9 @@ package com.taobao.tddl.dbsync.binlog;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
|
||||
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
|
||||
|
||||
/**
|
||||
@@ -20,6 +22,10 @@ public final class LogContext {
|
||||
|
||||
private LogPosition logPosition;
|
||||
|
||||
private GTIDSet gtidSet;
|
||||
|
||||
private GtidLogEvent gtidLogEvent; // save current gtid log event
|
||||
|
||||
public LogContext(){
|
||||
this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
|
||||
}
|
||||
@@ -60,4 +66,20 @@ public final class LogContext {
|
||||
formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
|
||||
mapOfTable.clear();
|
||||
}
|
||||
|
||||
public GTIDSet getGtidSet() {
|
||||
return gtidSet;
|
||||
}
|
||||
|
||||
public void setGtidSet(GTIDSet gtidSet) {
|
||||
this.gtidSet = gtidSet;
|
||||
}
|
||||
|
||||
public GtidLogEvent getGtidLogEvent() {
|
||||
return gtidLogEvent;
|
||||
}
|
||||
|
||||
public void setGtidLogEvent(GtidLogEvent gtidLogEvent) {
|
||||
this.gtidLogEvent = gtidLogEvent;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import java.util.BitSet;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
|
||||
import com.taobao.tddl.dbsync.binlog.event.AppendBlockLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.BeginLoadQueryLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.CreateFileLogEvent;
|
||||
@@ -46,7 +47,7 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.StartEncryptionLogEvent;
|
||||
|
||||
/**
|
||||
* Implements a binary-log decoder.
|
||||
*
|
||||
*
|
||||
* <pre>
|
||||
* LogDecoder decoder = new LogDecoder();
|
||||
* decoder.handle(...);
|
||||
@@ -61,7 +62,7 @@ import com.taobao.tddl.dbsync.binlog.event.mariadb.StartEncryptionLogEvent;
|
||||
* while (event != null);
|
||||
* // no more events in buffer.
|
||||
* </pre>
|
||||
*
|
||||
*
|
||||
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||
* @version 1.0
|
||||
*/
|
||||
@@ -88,7 +89,7 @@ public final class LogDecoder {
|
||||
|
||||
/**
|
||||
* Decoding an event from binary-log buffer.
|
||||
*
|
||||
*
|
||||
* @return <code>UknownLogEvent</code> if event type is unknown or skipped,
|
||||
* <code>null</code> if buffer is not including a full event.
|
||||
*/
|
||||
@@ -140,7 +141,7 @@ public final class LogDecoder {
|
||||
|
||||
/**
|
||||
* Deserialize an event from buffer.
|
||||
*
|
||||
*
|
||||
* @return <code>UknownLogEvent</code> if event type is unknown or skipped.
|
||||
*/
|
||||
public static LogEvent decode(LogBuffer buffer, LogHeader header, LogContext context) throws IOException {
|
||||
@@ -159,18 +160,21 @@ public final class LogDecoder {
|
||||
// remove checksum bytes
|
||||
buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
|
||||
}
|
||||
|
||||
GTIDSet gtidSet = context.getGtidSet();
|
||||
GtidLogEvent gtidLogEvent = context.getGtidLogEvent();
|
||||
switch (header.getType()) {
|
||||
case LogEvent.QUERY_EVENT: {
|
||||
QueryLogEvent event = new QueryLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.XID_EVENT: {
|
||||
XidLogEvent event = new XidLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.TABLE_MAP_EVENT: {
|
||||
@@ -268,12 +272,14 @@ public final class LogDecoder {
|
||||
RandLogEvent event = new RandLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.USER_VAR_EVENT: {
|
||||
UserVarLogEvent event = new UserVarLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.FORMAT_DESCRIPTION_EVENT: {
|
||||
@@ -336,6 +342,7 @@ public final class LogDecoder {
|
||||
RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.WRITE_ROWS_EVENT: {
|
||||
@@ -343,6 +350,7 @@ public final class LogDecoder {
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
event.fillTable(context);
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.UPDATE_ROWS_EVENT: {
|
||||
@@ -350,6 +358,7 @@ public final class LogDecoder {
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
event.fillTable(context);
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.DELETE_ROWS_EVENT: {
|
||||
@@ -357,6 +366,7 @@ public final class LogDecoder {
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
event.fillTable(context);
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.GTID_LOG_EVENT:
|
||||
@@ -364,6 +374,13 @@ public final class LogDecoder {
|
||||
GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
if (gtidSet != null) {
|
||||
gtidSet.update(event.getGtidStr());
|
||||
// update latest gtid
|
||||
header.putGtid(gtidSet, event);
|
||||
}
|
||||
// update current gtid event to context
|
||||
context.setGtidLogEvent(event);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: {
|
||||
@@ -394,6 +411,7 @@ public final class LogDecoder {
|
||||
AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
|
||||
/* updating position in context */
|
||||
logPosition.position = header.getLogPos();
|
||||
header.putGtid(context.getGtidSet(), gtidLogEvent);
|
||||
return event;
|
||||
}
|
||||
case LogEvent.BINLOG_CHECKPOINT_EVENT: {
|
||||
|
||||
@@ -14,10 +14,10 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||
public class GtidLogEvent extends LogEvent {
|
||||
|
||||
// / Length of the commit_flag in event encoding
|
||||
public static final int ENCODED_FLAG_LENGTH = 1;
|
||||
public static final int ENCODED_FLAG_LENGTH = 1;
|
||||
// / Length of SID in event encoding
|
||||
public static final int ENCODED_SID_LENGTH = 16;
|
||||
public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2;
|
||||
public static final int ENCODED_SID_LENGTH = 16;
|
||||
public static final int LOGICAL_TIMESTAMP_TYPE_CODE = 2;
|
||||
|
||||
private boolean commitFlag;
|
||||
private UUID sid;
|
||||
@@ -50,8 +50,6 @@ public class GtidLogEvent extends LogEvent {
|
||||
sequenceNumber = buffer.getLong64();
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ignore gtid info read
|
||||
// sid.copy_from((uchar *)ptr_buffer);
|
||||
// ptr_buffer+= ENCODED_SID_LENGTH;
|
||||
@@ -82,4 +80,11 @@ public class GtidLogEvent extends LogEvent {
|
||||
public Long getSequenceNumber() {
|
||||
return sequenceNumber;
|
||||
}
|
||||
|
||||
public String getGtidStr() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(sid.toString()).append(":");
|
||||
sb.append(gno);
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package com.taobao.tddl.dbsync.binlog.event;
|
||||
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
|
||||
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* The Common-Header, documented in the table @ref Table_common_header "below",
|
||||
* always has the same form and length within one version of MySQL. Each event
|
||||
@@ -120,6 +124,13 @@ public final class LogHeader {
|
||||
*/
|
||||
protected String logFileName;
|
||||
|
||||
protected Map<String, String> gtidMap = new HashMap<>();
|
||||
|
||||
private static final String CURRENT_GTID_STRING = "curt_gtid";
|
||||
private static final String CURRENT_GTID_SN = "curt_gtid_sn";
|
||||
private static final String CURRENT_GTID_LAST_COMMIT = "curt_gtid_lct";
|
||||
private static final String GTID_SET_STRING = "gtid_str";
|
||||
|
||||
/* for Start_event_v3 */
|
||||
public LogHeader(final int type){
|
||||
this.type = type;
|
||||
@@ -288,4 +299,31 @@ public final class LogHeader {
|
||||
crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
|
||||
}
|
||||
}
|
||||
|
||||
public String getGtidSetStr() {
|
||||
return gtidMap.get(GTID_SET_STRING);
|
||||
}
|
||||
|
||||
public String getCurrentGtid() {
|
||||
return gtidMap.get(CURRENT_GTID_STRING);
|
||||
}
|
||||
|
||||
public String getCurrentGtidSn() {
|
||||
return gtidMap.get(CURRENT_GTID_SN);
|
||||
}
|
||||
|
||||
public String getCurrentGtidLastCommit() {
|
||||
return gtidMap.get(CURRENT_GTID_LAST_COMMIT);
|
||||
}
|
||||
|
||||
public void putGtid(GTIDSet gtidSet, GtidLogEvent event) {
|
||||
if (gtidSet != null) {
|
||||
gtidMap.put(GTID_SET_STRING, gtidSet.toString());
|
||||
if (event != null) {
|
||||
gtidMap.put(CURRENT_GTID_STRING, event.getGtidStr());
|
||||
gtidMap.put(CURRENT_GTID_SN, String.valueOf(event.getSequenceNumber()));
|
||||
gtidMap.put(CURRENT_GTID_LAST_COMMIT, String.valueOf(event.getLastCommitted()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,9 +15,7 @@ public class FetcherPerformanceTest {
|
||||
DirectLogFetcher fetcher = new DirectLogFetcher();
|
||||
try {
|
||||
Class.forName("com.mysql.jdbc.Driver");
|
||||
Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306",
|
||||
"root",
|
||||
"hello");
|
||||
Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "root", "hello");
|
||||
Statement statement = connection.createStatement();
|
||||
statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
|
||||
statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
|
||||
|
||||
@@ -387,7 +387,7 @@ public class CanalController {
|
||||
return config;
|
||||
}
|
||||
|
||||
private String getProperty(Properties properties, String key) {
|
||||
public String getProperty(Properties properties, String key) {
|
||||
key = StringUtils.trim(key);
|
||||
String value = System.getProperty(key);
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import com.alibaba.otter.canal.server.CanalMQStarter;
|
||||
import com.alibaba.otter.canal.spi.CanalMQProducer;
|
||||
import java.io.FileInputStream;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -57,7 +56,7 @@ public class CanalLauncher {
|
||||
});
|
||||
|
||||
CanalMQProducer canalMQProducer = null;
|
||||
String serverMode = properties.getProperty(CanalConstants.CANAL_SERVER_MODE, "tcp");
|
||||
String serverMode = controller.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
|
||||
if (serverMode.equalsIgnoreCase("kafka")) {
|
||||
canalMQProducer = new CanalKafkaProducer();
|
||||
} else if (serverMode.equalsIgnoreCase("rocketmq")) {
|
||||
@@ -68,6 +67,7 @@ public class CanalLauncher {
|
||||
if (canalServerStarter != null) {
|
||||
canalServerStarter.init();
|
||||
}
|
||||
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.error("## Something goes wrong when starting up the canal Server:", e);
|
||||
|
||||
@@ -1,4 +1,34 @@
|
||||
{
|
||||
"__inputs": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus"
|
||||
}
|
||||
],
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"version": "5.2.2"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "graph",
|
||||
"name": "Graph",
|
||||
"version": "5.0.0"
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "5.0.0"
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
@@ -15,8 +45,8 @@
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": 8,
|
||||
"iteration": 1534740332233,
|
||||
"id": null,
|
||||
"iteration": 1536989235272,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
@@ -189,7 +219,7 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\"}[2m]) / 1024",
|
||||
"expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
@@ -203,6 +233,22 @@
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "outbound",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "inbound-1",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "inbound-2",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
@@ -399,20 +445,36 @@
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
|
||||
"expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "dump",
|
||||
"refId": "A"
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
|
||||
"expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "sink",
|
||||
"refId": "B"
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "dump-1",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "dump-2",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
@@ -465,7 +527,7 @@
|
||||
},
|
||||
"id": 32,
|
||||
"panels": [],
|
||||
"title": "Server",
|
||||
"title": "Throughput",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
@@ -660,180 +722,6 @@
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$datasource",
|
||||
"description": "Canal instance ringbuffer内未释放的events数量。",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 7
|
||||
},
|
||||
"id": 20,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "events",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Store remain events",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"decimals": null,
|
||||
"format": "none",
|
||||
"label": "",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"decimals": null,
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$datasource",
|
||||
"description": "Canal instance ringbuffer 内未释放events占用内存。",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 7
|
||||
},
|
||||
"id": 22,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "memsize",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Store remain mem",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "deckbytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
@@ -938,7 +826,7 @@
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$datasource",
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"description": "client 请求的GET与ACK包的QPS。",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
@@ -1218,6 +1106,193 @@
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"collapsed": false,
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"id": 36,
|
||||
"panels": [],
|
||||
"title": "Store",
|
||||
"type": "row"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$datasource",
|
||||
"description": "Canal instance ringbuffer内未释放的events数量。",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 19
|
||||
},
|
||||
"id": 20,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "events",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Store remain events",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"decimals": null,
|
||||
"format": "none",
|
||||
"label": "",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"decimals": null,
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$datasource",
|
||||
"description": "Canal instance ringbuffer 内未释放events占用内存。",
|
||||
"fill": 1,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 19
|
||||
},
|
||||
"id": 22,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",
|
||||
"format": "time_series",
|
||||
"interval": "15s",
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "memsize",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Store remain mem",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "deckbytes",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": false,
|
||||
@@ -1230,9 +1305,8 @@
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"tags": [],
|
||||
"text": "Prometheus",
|
||||
"value": "Prometheus"
|
||||
"text": "prometheus",
|
||||
"value": "prometheus"
|
||||
},
|
||||
"hide": 0,
|
||||
"label": "datasource",
|
||||
@@ -1245,10 +1319,7 @@
|
||||
},
|
||||
{
|
||||
"allValue": null,
|
||||
"current": {
|
||||
"text": "example",
|
||||
"value": "example"
|
||||
},
|
||||
"current": {},
|
||||
"datasource": "$datasource",
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
@@ -1269,7 +1340,7 @@
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
@@ -1300,5 +1371,5 @@
|
||||
"timezone": "",
|
||||
"title": "Canal instances",
|
||||
"uid": "8vh8NGpiz",
|
||||
"version": 1
|
||||
"version": 103
|
||||
}
|
||||
@@ -3,17 +3,21 @@ retries: 0
|
||||
batchSize: 16384
|
||||
lingerMs: 1
|
||||
bufferMemory: 33554432
|
||||
# canal的批次大小，单位 k，量大建议改为1M
|
||||
|
||||
# Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下)
|
||||
canalBatchSize: 50
|
||||
filterTransactionEntry: true
|
||||
# Canal get数据的超时时间, 单位: 毫秒, 空为不限超时
|
||||
canalGetTimeout: 100
|
||||
flatMessage: true
|
||||
|
||||
canalDestinations:
|
||||
- canalDestination: example
|
||||
topic: example
|
||||
partition: 1
|
||||
# 一个destination可以对应多个topic
|
||||
#topics:
|
||||
# - topic: example
|
||||
# partition:
|
||||
# #对应topic分区数量
|
||||
# partitionsNum: 3
|
||||
# partitionHash:
|
||||
# #库名.表名: 唯一主键
|
||||
# mytest.person: id
|
||||
|
||||
|
||||
|
||||
@@ -26,10 +26,10 @@ CREATE TABLE IF NOT EXISTS `meta_history` (
|
||||
`binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id',
|
||||
`binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳',
|
||||
`use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema',
|
||||
`schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
|
||||
`table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
|
||||
`sql` longtext DEFAULT NULL COMMENT '执行的sql',
|
||||
`type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
|
||||
`sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema',
|
||||
`sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table',
|
||||
`sql_text` longtext DEFAULT NULL COMMENT '执行的sql',
|
||||
`sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型',
|
||||
`extra` text DEFAULT NULL COMMENT '额外的扩展信息',
|
||||
PRIMARY KEY (`id`),
|
||||
UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`),
|
||||
|
||||
+10
@@ -138,6 +138,16 @@ public class BioSocketChannel implements SocketChannel {
|
||||
if (socket != null) {
|
||||
return socket.getRemoteSocketAddress();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public SocketAddress getLocalSocketAddress() {
|
||||
Socket socket = this.socket;
|
||||
if (socket != null) {
|
||||
return socket.getLocalSocketAddress();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
+4
@@ -216,6 +216,10 @@ public class NettySocketChannel implements SocketChannel {
|
||||
return channel != null ? channel.remoteAddress() : null;
|
||||
}
|
||||
|
||||
public SocketAddress getLocalSocketAddress() {
|
||||
return channel != null ? channel.localAddress() : null;
|
||||
}
|
||||
|
||||
public void close() {
|
||||
if (channel != null) {
|
||||
channel.close();
|
||||
|
||||
+2
@@ -21,5 +21,7 @@ public interface SocketChannel {
|
||||
|
||||
public SocketAddress getRemoteSocketAddress();
|
||||
|
||||
public SocketAddress getLocalSocketAddress();
|
||||
|
||||
public void close();
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.SystemUtils;
|
||||
import org.slf4j.Logger;
|
||||
@@ -299,4 +300,55 @@ public class AbstractCanalClientTest {
|
||||
this.connector = connector;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前Entry的 GTID信息示例
|
||||
* @param header
|
||||
* @return
|
||||
*/
|
||||
public static String getCurrentGtid(CanalEntry.Header header) {
|
||||
List<CanalEntry.Pair> props = header.getPropsList();
|
||||
if (props != null && props.size() > 0) {
|
||||
for (CanalEntry.Pair pair : props) {
|
||||
if ("curtGtid".equals(pair.getKey())) {
|
||||
return pair.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前Entry的 GTID Sequence No信息示例
|
||||
* @param header
|
||||
* @return
|
||||
*/
|
||||
public static String getCurrentGtidSn(CanalEntry.Header header) {
|
||||
List<CanalEntry.Pair> props = header.getPropsList();
|
||||
if (props != null && props.size() > 0) {
|
||||
for (CanalEntry.Pair pair : props) {
|
||||
if ("curtGtidSn".equals(pair.getKey())) {
|
||||
return pair.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前Entry的 GTID Last Committed信息示例
|
||||
* @param header
|
||||
* @return
|
||||
*/
|
||||
public static String getCurrentGtidLct(CanalEntry.Header header) {
|
||||
List<CanalEntry.Pair> props = header.getPropsList();
|
||||
if (props != null && props.size() > 0) {
|
||||
for (CanalEntry.Pair pair : props) {
|
||||
if ("curtGtidLct".equals(pair.getKey())) {
|
||||
return pair.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
-5
@@ -8,23 +8,23 @@ public class PositionNotFoundException extends CanalParseException {
|
||||
|
||||
private static final long serialVersionUID = -7382448928116244017L;
|
||||
|
||||
public PositionNotFoundException(String errorCode) {
|
||||
public PositionNotFoundException(String errorCode){
|
||||
super(errorCode);
|
||||
}
|
||||
|
||||
public PositionNotFoundException(String errorCode, Throwable cause) {
|
||||
public PositionNotFoundException(String errorCode, Throwable cause){
|
||||
super(errorCode, cause);
|
||||
}
|
||||
|
||||
public PositionNotFoundException(String errorCode, String errorDesc) {
|
||||
public PositionNotFoundException(String errorCode, String errorDesc){
|
||||
super(errorCode, errorDesc);
|
||||
}
|
||||
|
||||
public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause) {
|
||||
public PositionNotFoundException(String errorCode, String errorDesc, Throwable cause){
|
||||
super(errorCode, errorDesc, cause);
|
||||
}
|
||||
|
||||
public PositionNotFoundException(Throwable cause) {
|
||||
public PositionNotFoundException(Throwable cause){
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
|
||||
+6
-6
@@ -6,27 +6,27 @@ import com.alibaba.otter.canal.common.CanalException;
|
||||
* @author chengjin.lyf on 2018/8/8 下午1:07
|
||||
* @since 1.0.25
|
||||
*/
|
||||
public class ServerIdNotMatchException extends CanalException{
|
||||
public class ServerIdNotMatchException extends CanalException {
|
||||
|
||||
private static final long serialVersionUID = -6124989280379293953L;
|
||||
|
||||
public ServerIdNotMatchException(String errorCode) {
|
||||
public ServerIdNotMatchException(String errorCode){
|
||||
super(errorCode);
|
||||
}
|
||||
|
||||
public ServerIdNotMatchException(String errorCode, Throwable cause) {
|
||||
public ServerIdNotMatchException(String errorCode, Throwable cause){
|
||||
super(errorCode, cause);
|
||||
}
|
||||
|
||||
public ServerIdNotMatchException(String errorCode, String errorDesc) {
|
||||
public ServerIdNotMatchException(String errorCode, String errorDesc){
|
||||
super(errorCode, errorDesc);
|
||||
}
|
||||
|
||||
public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause) {
|
||||
public ServerIdNotMatchException(String errorCode, String errorDesc, Throwable cause){
|
||||
super(errorCode, errorDesc, cause);
|
||||
}
|
||||
|
||||
public ServerIdNotMatchException(Throwable cause) {
|
||||
public ServerIdNotMatchException(Throwable cause){
|
||||
super(cause);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,11 +19,13 @@ import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
|
||||
import com.alibaba.otter.canal.filter.CanalEventFilter;
|
||||
import com.alibaba.otter.canal.parse.CanalEventParser;
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
|
||||
import com.alibaba.otter.canal.parse.exception.CanalParseException;
|
||||
import com.alibaba.otter.canal.parse.exception.PositionNotFoundException;
|
||||
import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
|
||||
import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
|
||||
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlMultiStageCoprocessor;
|
||||
import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
|
||||
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
@@ -235,12 +237,14 @@ public abstract class AbstractEventParser<EVENT> extends AbstractCanalLifeCycle
|
||||
if (parallel) {
|
||||
// build stage processor
|
||||
multiStageCoprocessor = buildMultiStageCoprocessor();
|
||||
multiStageCoprocessor.start();
|
||||
|
||||
if (isGTIDMode()) {
|
||||
// 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
|
||||
erosaConnection.dump(MysqlGTIDSet.parse(startPosition.getGtid()), multiStageCoprocessor);
|
||||
GTIDSet gtidSet = MysqlGTIDSet.parse(startPosition.getGtid());
|
||||
((MysqlMultiStageCoprocessor) multiStageCoprocessor).setGtidSet(gtidSet);
|
||||
multiStageCoprocessor.start();
|
||||
erosaConnection.dump(gtidSet, multiStageCoprocessor);
|
||||
} else {
|
||||
multiStageCoprocessor.start();
|
||||
if (StringUtils.isEmpty(startPosition.getJournalName())
|
||||
&& startPosition.getTimestamp() != null) {
|
||||
erosaConnection.dump(startPosition.getTimestamp(), multiStageCoprocessor);
|
||||
|
||||
@@ -5,5 +5,6 @@ package com.alibaba.otter.canal.parse.inbound;
|
||||
* @since 1.0.25
|
||||
*/
|
||||
public interface ParserExceptionHandler {
|
||||
|
||||
void handle(Throwable e);
|
||||
}
|
||||
|
||||
+6
-8
@@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory;
|
||||
import com.alibaba.otter.canal.filter.CanalEventFilter;
|
||||
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
|
||||
import com.alibaba.otter.canal.parse.CanalEventParser;
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet;
|
||||
import com.alibaba.otter.canal.parse.exception.CanalParseException;
|
||||
import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
|
||||
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
|
||||
@@ -39,6 +38,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
|
||||
protected boolean filterRows = false;
|
||||
protected boolean filterTableError = false;
|
||||
protected boolean useDruidDdlFilter = true;
|
||||
// instance received binlog bytes
|
||||
protected final AtomicLong receivedBinlogBytes = new AtomicLong(0L);
|
||||
private final AtomicLong eventsPublishBlockingTime = new AtomicLong(0L);
|
||||
|
||||
protected BinlogParser buildParser() {
|
||||
@@ -76,13 +77,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
|
||||
* @return
|
||||
*/
|
||||
protected boolean processTableMeta(EntryPosition position) {
|
||||
if (isGTIDMode()) {
|
||||
if (binlogParser instanceof LogEventConvert) {
|
||||
// 记录gtid
|
||||
((LogEventConvert) binlogParser).setGtidSet(MysqlGTIDSet.parse(position.getGtid()));
|
||||
}
|
||||
}
|
||||
|
||||
if (tableMetaTSDB != null) {
|
||||
if (position.getTimestamp() == null || position.getTimestamp() <= 0) {
|
||||
throw new CanalParseException("use gtid and TableMeta TSDB should be config timestamp > 0");
|
||||
@@ -212,4 +206,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser {
|
||||
return this.eventsPublishBlockingTime;
|
||||
}
|
||||
|
||||
public AtomicLong getReceivedBinlogBytes() {
|
||||
return this.receivedBinlogBytes;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+13
-2
@@ -4,6 +4,7 @@ import static com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetche
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -179,6 +180,8 @@ public class MysqlConnection implements ErosaConnection {
|
||||
fetcher.start(connector.getChannel());
|
||||
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
|
||||
LogContext context = new LogContext();
|
||||
// fix bug: #890 将gtid传输至context中，供decode使用
|
||||
context.setGtidSet(gtidSet);
|
||||
while (fetcher.fetch()) {
|
||||
accumulateReceivedBytes(fetcher.limit());
|
||||
LogEvent event = null;
|
||||
@@ -252,10 +255,18 @@ public class MysqlConnection implements ErosaConnection {
|
||||
|
||||
private void sendRegisterSlave() throws IOException {
|
||||
RegisterSlaveCommandPacket cmd = new RegisterSlaveCommandPacket();
|
||||
cmd.reportHost = authInfo.getAddress().getAddress().getHostAddress();
|
||||
SocketAddress socketAddress = connector.getChannel().getLocalSocketAddress();
|
||||
if (socketAddress == null || !(socketAddress instanceof InetSocketAddress)) {
|
||||
return;
|
||||
}
|
||||
|
||||
InetSocketAddress address = (InetSocketAddress) socketAddress;
|
||||
String host = address.getHostString();
|
||||
int port = address.getPort();
|
||||
cmd.reportHost = host;
|
||||
cmd.reportPort = port;
|
||||
cmd.reportPasswd = authInfo.getPassword();
|
||||
cmd.reportUser = authInfo.getUsername();
|
||||
cmd.reportPort = authInfo.getAddress().getPort(); // 暂时先用master节点的port
|
||||
cmd.serverId = this.slaveId;
|
||||
byte[] cmdBody = cmd.toBytes();
|
||||
|
||||
|
||||
+12
-20
@@ -8,7 +8,6 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TimerTask;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
@@ -49,30 +48,27 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
|
||||
|
||||
private CanalHAController haController = null;
|
||||
|
||||
private int defaultConnectionTimeoutInSeconds = 30; // sotimeout
|
||||
private int defaultConnectionTimeoutInSeconds = 30; // sotimeout
|
||||
private int receiveBufferSize = 64 * 1024;
|
||||
private int sendBufferSize = 64 * 1024;
|
||||
// 数据库信息
|
||||
protected AuthenticationInfo masterInfo; // 主库
|
||||
protected AuthenticationInfo standbyInfo; // 备库
|
||||
protected AuthenticationInfo masterInfo; // 主库
|
||||
protected AuthenticationInfo standbyInfo; // 备库
|
||||
// binlog信息
|
||||
protected EntryPosition masterPosition;
|
||||
protected EntryPosition standbyPosition;
|
||||
private long slaveId; // 链接到mysql的slave
|
||||
private long slaveId; // 链接到mysql的slave
|
||||
// 心跳检查信息
|
||||
private String detectingSQL; // 心跳sql
|
||||
private MysqlConnection metaConnection; // 查询meta信息的链接
|
||||
private TableMetaCache tableMetaCache; // 对应meta
|
||||
private int fallbackIntervalInSeconds = 60; // 切换回退时间
|
||||
private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验
|
||||
private BinlogImage[] supportBinlogImages; // 支持的binlogImage,如果设置会执行强校验
|
||||
private String detectingSQL; // 心跳sql
|
||||
private MysqlConnection metaConnection; // 查询meta信息的链接
|
||||
private TableMetaCache tableMetaCache; // 对应meta
|
||||
private int fallbackIntervalInSeconds = 60; // 切换回退时间
|
||||
private BinlogFormat[] supportBinlogFormats; // 支持的binlogFormat,如果设置会执行强校验
|
||||
private BinlogImage[] supportBinlogImages; // 支持的binlogImage,如果设置会执行强校验
|
||||
|
||||
// update by yishun.chen,特殊异常处理参数
|
||||
private int dumpErrorCount = 0; // binlogDump失败异常计数
|
||||
private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值
|
||||
|
||||
// instance received binlog bytes
|
||||
private final AtomicLong receivedBinlogBytes = new AtomicLong(0L);
|
||||
private int dumpErrorCount = 0; // binlogDump失败异常计数
|
||||
private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值
|
||||
|
||||
protected ErosaConnection buildErosaConnection() {
|
||||
return buildMysqlConnection(this.runningInfo);
|
||||
@@ -907,8 +903,4 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
|
||||
this.dumpErrorCountThreshold = dumpErrorCountThreshold;
|
||||
}
|
||||
|
||||
public AtomicLong getReceivedBinlogBytes() {
|
||||
return this.receivedBinlogBytes;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+28
-5
@@ -2,11 +2,13 @@ package com.alibaba.otter.canal.parse.inbound.mysql;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.locks.LockSupport;
|
||||
|
||||
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
|
||||
import com.alibaba.otter.canal.parse.exception.CanalParseException;
|
||||
import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
|
||||
import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer;
|
||||
@@ -35,6 +37,7 @@ import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
|
||||
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
|
||||
|
||||
|
||||
/**
|
||||
* 针对解析器提供一个多阶段协同的处理
|
||||
*
|
||||
@@ -63,6 +66,10 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
private String destination;
|
||||
private volatile CanalParseException exception;
|
||||
private AtomicLong eventsPublishBlockingTime;
|
||||
private GTIDSet gtidSet;
|
||||
private WorkerPool<MessageEvent> workerPool;
|
||||
private BatchEventProcessor<MessageEvent> simpleParserStage;
|
||||
private BatchEventProcessor<MessageEvent> sinkStoreStage;
|
||||
|
||||
public MysqlMultiStageCoprocessor(int ringBufferSize, int parserThreadCount, LogEventConvert logEventConvert,
|
||||
EventTransactionBuffer transactionBuffer, String destination){
|
||||
@@ -81,15 +88,15 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
ringBufferSize,
|
||||
new BlockingWaitStrategy());
|
||||
int tc = parserThreadCount > 0 ? parserThreadCount : 1;
|
||||
this.parserExecutor = Executors.newFixedThreadPool(tc,
|
||||
new NamedThreadFactory("MultiStageCoprocessor-Parser-" + destination));
|
||||
this.parserExecutor = Executors.newFixedThreadPool(tc, new NamedThreadFactory("MultiStageCoprocessor-Parser-"
|
||||
+ destination));
|
||||
|
||||
this.stageExecutor = Executors.newFixedThreadPool(2, new NamedThreadFactory("MultiStageCoprocessor-other-"
|
||||
+ destination));
|
||||
SequenceBarrier sequenceBarrier = disruptorMsgBuffer.newBarrier();
|
||||
ExceptionHandler exceptionHandler = new SimpleFatalExceptionHandler();
|
||||
// stage 2
|
||||
BatchEventProcessor<MessageEvent> simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
|
||||
simpleParserStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
|
||||
sequenceBarrier,
|
||||
new SimpleParserStage());
|
||||
simpleParserStage.setExceptionHandler(exceptionHandler);
|
||||
@@ -101,7 +108,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
for (int i = 0; i < parserThreadCount; i++) {
|
||||
workHandlers[i] = new DmlParserStage();
|
||||
}
|
||||
WorkerPool<MessageEvent> workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer,
|
||||
workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer,
|
||||
dmlParserSequenceBarrier,
|
||||
exceptionHandler,
|
||||
workHandlers);
|
||||
@@ -110,7 +117,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
|
||||
// stage 4
|
||||
SequenceBarrier sinkSequenceBarrier = disruptorMsgBuffer.newBarrier(sequence);
|
||||
BatchEventProcessor<MessageEvent> sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
|
||||
sinkStoreStage = new BatchEventProcessor<MessageEvent>(disruptorMsgBuffer,
|
||||
sinkSequenceBarrier,
|
||||
new SinkStoreStage());
|
||||
sinkStoreStage.setExceptionHandler(exceptionHandler);
|
||||
@@ -124,14 +131,24 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
// fix bug #968，对于pool与
|
||||
workerPool.halt();
|
||||
simpleParserStage.halt();
|
||||
sinkStoreStage.halt();
|
||||
try {
|
||||
parserExecutor.shutdownNow();
|
||||
while (!parserExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
|
||||
parserExecutor.shutdownNow();
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
try {
|
||||
stageExecutor.shutdownNow();
|
||||
while (!stageExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
|
||||
stageExecutor.shutdownNow();
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
// ignore
|
||||
}
|
||||
@@ -227,6 +244,9 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
public SimpleParserStage(){
|
||||
decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
|
||||
context = new LogContext();
|
||||
if (gtidSet != null) {
|
||||
context.setGtidSet(gtidSet);
|
||||
}
|
||||
}
|
||||
|
||||
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
|
||||
@@ -447,4 +467,7 @@ public class MysqlMultiStageCoprocessor extends AbstractCanalLifeCycle implement
|
||||
this.eventsPublishBlockingTime = eventsPublishBlockingTime;
|
||||
}
|
||||
|
||||
public void setGtidSet(GTIDSet gtidSet) {
|
||||
this.gtidSet = gtidSet;
|
||||
}
|
||||
}
|
||||
|
||||
+20
-21
@@ -17,7 +17,6 @@ import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||
import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
|
||||
import com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet;
|
||||
import com.alibaba.otter.canal.parse.exception.CanalParseException;
|
||||
import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
|
||||
import com.alibaba.otter.canal.parse.inbound.BinlogParser;
|
||||
@@ -99,13 +98,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
private boolean filterRows = false;
|
||||
private boolean useDruidDdlFilter = true;
|
||||
|
||||
// latest gtid
|
||||
private GTIDSet gtidSet;
|
||||
|
||||
public LogEventConvert(GTIDSet gtidSet){
|
||||
this.gtidSet = gtidSet;
|
||||
}
|
||||
|
||||
public LogEventConvert(){
|
||||
|
||||
}
|
||||
@@ -172,13 +164,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
|
||||
private Entry parseGTIDLogEvent(GtidLogEvent logEvent) {
|
||||
LogHeader logHeader = logEvent.getHeader();
|
||||
String value = logEvent.getSid().toString() + ":" + logEvent.getGno();
|
||||
Pair.Builder builder = Pair.newBuilder();
|
||||
builder.setKey("gtid");
|
||||
builder.setValue(value);
|
||||
if (gtidSet != null) {
|
||||
gtidSet.update(value);
|
||||
}
|
||||
builder.setValue(logEvent.getGtidStr());
|
||||
|
||||
if (logEvent.getLastCommitted() != null) {
|
||||
builder.setKey("lastCommitted");
|
||||
builder.setValue(String.valueOf(logEvent.getLastCommitted()));
|
||||
@@ -556,7 +545,6 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
rowsCount++;
|
||||
rowChangeBuider.addRowDatas(rowDataBuilder.build());
|
||||
}
|
||||
|
||||
TableMapLogEvent table = event.getTable();
|
||||
Header header = createHeader(event.getHeader(),
|
||||
table.getDbName(),
|
||||
@@ -845,9 +833,24 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
}
|
||||
headerBuilder.setEventLength(logHeader.getEventLen());
|
||||
// enable gtid position
|
||||
if (gtidSet != null) {
|
||||
String gtid = gtidSet.toString();
|
||||
headerBuilder.setGtid(gtid);
|
||||
if (StringUtils.isNotEmpty(logHeader.getGtidSetStr())) {
|
||||
headerBuilder.setGtid(logHeader.getGtidSetStr());
|
||||
}
|
||||
// add current gtid
|
||||
if (StringUtils.isNotEmpty(logHeader.getCurrentGtid())) {
|
||||
Pair pair = createSpecialPair("curtGtid", logHeader.getCurrentGtid());
|
||||
headerBuilder.addProps(pair);
|
||||
}
|
||||
// add current gtid sequence no
|
||||
if (StringUtils.isNotEmpty(logHeader.getCurrentGtidSn())) {
|
||||
Pair pair = createSpecialPair("curtGtidSn", logHeader.getCurrentGtidSn());
|
||||
headerBuilder.addProps(pair);
|
||||
}
|
||||
|
||||
// add current gtid last committed
|
||||
if (StringUtils.isNotEmpty(logHeader.getCurrentGtidLastCommit())) {
|
||||
Pair pair = createSpecialPair("curtGtidLct", logHeader.getCurrentGtidLastCommit());
|
||||
headerBuilder.addProps(pair);
|
||||
}
|
||||
|
||||
// add rowsCount suppport
|
||||
@@ -977,8 +980,4 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
|
||||
public void setFilterRows(boolean filterRows) {
|
||||
this.filterRows = filterRows;
|
||||
}
|
||||
|
||||
public void setGtidSet(GTIDSet gtidSet) {
|
||||
this.gtidSet = gtidSet;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -44,8 +44,8 @@ public class SimpleDdlParser {
|
||||
* http://dev.mysql.com/doc/refman/5.6/en/create-index.html
|
||||
* </pre>
|
||||
*/
|
||||
public static final String CREATE_INDEX_PATTERN = "^\\s*CREATE\\s*(UNIQUE)?(FULLTEXT)?(SPATIAL)?\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
|
||||
public static final String DROP_INDEX_PATTERN = "^\\s*DROP\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
|
||||
public static final String CREATE_INDEX_PATTERN = "^\\s*CREATE\\s*(UNIQUE)?(FULLTEXT)?(SPATIAL)?\\s*INDEX\\s*(.*?)\\s+ON\\s+(.*?)$";
|
||||
public static final String DROP_INDEX_PATTERN = "^\\s*DROP\\s*INDEX\\s*(.*?)\\s+ON\\s+(.*?)$";
|
||||
|
||||
public static DdlResult parse(String queryString, String schmeaName) {
|
||||
queryString = removeComment(queryString); // 去除/* */的sql注释内容
|
||||
|
||||
+2
-2
@@ -20,12 +20,12 @@ public class MysqlBinlogDumpPerformanceTest {
|
||||
|
||||
public static void main(String args[]) {
|
||||
final MysqlEventParser controller = new MysqlEventParser();
|
||||
final EntryPosition startPosition = new EntryPosition("mysql-bin.001699", 120L, 100L);
|
||||
final EntryPosition startPosition = new EntryPosition("mysql-bin.000007", 89796293L, 100L);
|
||||
controller.setConnectionCharset(Charset.forName("UTF-8"));
|
||||
controller.setSlaveId(3344L);
|
||||
controller.setDetectingEnable(false);
|
||||
controller.setFilterQueryDml(true);
|
||||
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3328), "root", "hello"));
|
||||
controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("100.81.154.142", 3306), "canal", "canal"));
|
||||
controller.setMasterPosition(startPosition);
|
||||
controller.setEnableTsdb(false);
|
||||
controller.setDestination("example");
|
||||
|
||||
+14
@@ -225,5 +225,19 @@ public class SimpleDdlParserTest {
|
||||
Assert.assertNotNull(result);
|
||||
Assert.assertEquals("retl", result.getSchemaName());
|
||||
Assert.assertEquals("retl_mark", result.getTableName());
|
||||
|
||||
// test index name contains 'on' -- version
|
||||
queryString = "create index schema_new_index_version_s_idx on q_contract_account (contract_id,main_contract_id)";
|
||||
result = SimpleDdlParser.parse(queryString, "retl");
|
||||
Assert.assertNotNull(result);
|
||||
Assert.assertEquals("retl", result.getSchemaName());
|
||||
Assert.assertEquals("q_contract_account", result.getTableName());
|
||||
|
||||
queryString = "drop index schema_new_index_version_s_idx on q_contract_account";
|
||||
result = SimpleDdlParser.parse(queryString, "retl");
|
||||
Assert.assertNotNull(result);
|
||||
Assert.assertEquals("retl", result.getSchemaName());
|
||||
Assert.assertEquals("q_contract_account", result.getTableName());
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +237,7 @@
|
||||
<dependency>
|
||||
<groupId>com.google.protobuf</groupId>
|
||||
<artifactId>protobuf-java</artifactId>
|
||||
<version>2.6.1</version>
|
||||
<version>3.6.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.ibatis</groupId>
|
||||
|
||||
+3
-3
@@ -53,9 +53,9 @@ public class EntryCollector extends Collector implements InstanceRegistry {
|
||||
for (EntryMetricsHolder emh : instances.values()) {
|
||||
long now = System.currentTimeMillis();
|
||||
long latest = emh.latestExecTime.get();
|
||||
if (now >= latest) {
|
||||
delay.addMetric(emh.destLabelValues, (now - latest));
|
||||
}
|
||||
// execTime > now，delay显示为0
|
||||
long d = (now >= latest) ? (now - latest) : 0;
|
||||
delay.addMetric(emh.destLabelValues, d);
|
||||
transactions.addMetric(emh.destLabelValues, emh.transactionCounter.doubleValue());
|
||||
}
|
||||
mfs.add(delay);
|
||||
|
||||
+69
-27
@@ -1,14 +1,15 @@
|
||||
package com.alibaba.otter.canal.prometheus.impl;
|
||||
|
||||
import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST;
|
||||
import static com.alibaba.otter.canal.prometheus.CanalInstanceExports.DEST_LABELS_LIST;
|
||||
|
||||
import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
|
||||
import com.alibaba.otter.canal.parse.inbound.mysql.AbstractMysqlEventParser;
|
||||
import io.prometheus.client.Collector;
|
||||
import io.prometheus.client.CounterMetricFamily;
|
||||
import io.prometheus.client.GaugeMetricFamily;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
@@ -19,7 +20,6 @@ import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.instance.core.CanalInstance;
|
||||
import com.alibaba.otter.canal.parse.CanalEventParser;
|
||||
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
|
||||
import com.alibaba.otter.canal.prometheus.InstanceRegistry;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
@@ -34,11 +34,13 @@ public class ParserCollector extends Collector implements InstanceRegistry {
|
||||
private static final String RECEIVED_BINLOG = "canal_instance_received_binlog_bytes";
|
||||
private static final String PARSER_MODE = "canal_instance_parser_mode";
|
||||
private static final String MODE_LABEL = "parallel";
|
||||
private static final String PARSER_LABEL = "parser";
|
||||
private static final String PUBLISH_BLOCKING_HELP = "Publish blocking time of dump thread in milliseconds";
|
||||
private static final String RECEIVED_BINLOG_HELP = "Received binlog bytes";
|
||||
private static final String MODE_HELP = "Parser mode(parallel/serial) of instance";
|
||||
private final List<String> modeLabels = Arrays.asList(DEST, MODE_LABEL);
|
||||
private final ConcurrentMap<String, ParserMetricsHolder> instances = new ConcurrentHashMap<String, ParserMetricsHolder>();
|
||||
private final List<String> parserLabels = Arrays.asList(DEST, PARSER_LABEL);
|
||||
private final ConcurrentMap<String, ParserMetricsHolder> instances = new ConcurrentHashMap<>();
|
||||
|
||||
private ParserCollector() {}
|
||||
|
||||
@@ -52,53 +54,87 @@ public class ParserCollector extends Collector implements InstanceRegistry {
|
||||
|
||||
@Override
|
||||
public List<MetricFamilySamples> collect() {
|
||||
List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
|
||||
boolean hasParallel = false;
|
||||
List<MetricFamilySamples> mfs = new ArrayList<>();
|
||||
CounterMetricFamily bytesCounter = new CounterMetricFamily(RECEIVED_BINLOG,
|
||||
RECEIVED_BINLOG_HELP, DEST_LABELS_LIST);
|
||||
RECEIVED_BINLOG_HELP, parserLabels);
|
||||
GaugeMetricFamily modeGauge = new GaugeMetricFamily(PARSER_MODE,
|
||||
MODE_HELP, modeLabels);
|
||||
CounterMetricFamily blockingCounter = new CounterMetricFamily(PUBLISH_BLOCKING,
|
||||
PUBLISH_BLOCKING_HELP, DEST_LABELS_LIST);
|
||||
PUBLISH_BLOCKING_HELP, parserLabels);
|
||||
for (ParserMetricsHolder emh : instances.values()) {
|
||||
if (emh.isParallel) {
|
||||
blockingCounter.addMetric(emh.destLabelValues, (emh.eventsPublishBlockingTime.doubleValue() / NANO_PER_MILLI));
|
||||
hasParallel = true;
|
||||
if (emh instanceof GroupParserMetricsHolder) {
|
||||
GroupParserMetricsHolder group = (GroupParserMetricsHolder) emh;
|
||||
for (ParserMetricsHolder semh : group.holders) {
|
||||
singleCollect(bytesCounter, blockingCounter, modeGauge, semh);
|
||||
}
|
||||
}
|
||||
else {
|
||||
singleCollect(bytesCounter, blockingCounter, modeGauge, emh);
|
||||
}
|
||||
modeGauge.addMetric(emh.modeLabelValues, 1);
|
||||
bytesCounter.addMetric(emh.destLabelValues, emh.receivedBinlogBytes.doubleValue());
|
||||
|
||||
}
|
||||
mfs.add(bytesCounter);
|
||||
mfs.add(modeGauge);
|
||||
if (hasParallel) {
|
||||
if (!blockingCounter.samples.isEmpty()) {
|
||||
mfs.add(blockingCounter);
|
||||
}
|
||||
return mfs;
|
||||
}
|
||||
|
||||
private void singleCollect(CounterMetricFamily bytesCounter, CounterMetricFamily blockingCounter, GaugeMetricFamily modeGauge, ParserMetricsHolder holder) {
|
||||
if (holder.isParallel) {
|
||||
blockingCounter.addMetric(holder.parserLabelValues, (holder.eventsPublishBlockingTime.doubleValue() / NANO_PER_MILLI));
|
||||
}
|
||||
modeGauge.addMetric(holder.modeLabelValues, 1);
|
||||
bytesCounter.addMetric(holder.parserLabelValues, holder.receivedBinlogBytes.doubleValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void register(CanalInstance instance) {
|
||||
final String destination = instance.getDestination();
|
||||
ParserMetricsHolder holder = new ParserMetricsHolder();
|
||||
ParserMetricsHolder holder;
|
||||
CanalEventParser parser = instance.getEventParser();
|
||||
if (!(parser instanceof MysqlEventParser)) {
|
||||
throw new IllegalArgumentException("CanalEventParser must be MysqlEventParser");
|
||||
if (parser instanceof AbstractMysqlEventParser) {
|
||||
holder = singleHolder(destination, (AbstractMysqlEventParser)parser, "0");
|
||||
} else if (parser instanceof GroupEventParser) {
|
||||
holder = groupHolder(destination, (GroupEventParser)parser);
|
||||
} else {
|
||||
throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
|
||||
}
|
||||
MysqlEventParser mysqlParser = (MysqlEventParser) parser;
|
||||
holder.destLabelValues = Collections.singletonList(destination);
|
||||
holder.modeLabelValues = Arrays.asList(destination, Boolean.toString(mysqlParser.isParallel()));
|
||||
holder.eventsPublishBlockingTime = mysqlParser.getEventsPublishBlockingTime();
|
||||
holder.receivedBinlogBytes = mysqlParser.getReceivedBinlogBytes();
|
||||
holder.isParallel = mysqlParser.isParallel();
|
||||
Preconditions.checkNotNull(holder.eventsPublishBlockingTime);
|
||||
Preconditions.checkNotNull(holder.receivedBinlogBytes);
|
||||
Preconditions.checkNotNull(holder);
|
||||
ParserMetricsHolder old = instances.put(destination, holder);
|
||||
if (old != null) {
|
||||
logger.warn("Remove stale ParserCollector for instance {}.", destination);
|
||||
}
|
||||
}
|
||||
|
||||
private ParserMetricsHolder singleHolder(String destination, AbstractMysqlEventParser parser, String id) {
|
||||
ParserMetricsHolder holder = new ParserMetricsHolder();
|
||||
holder.parserLabelValues = Arrays.asList(destination, id);
|
||||
holder.modeLabelValues = Arrays.asList(destination, Boolean.toString(parser.isParallel()));
|
||||
holder.eventsPublishBlockingTime = parser.getEventsPublishBlockingTime();
|
||||
holder.receivedBinlogBytes = parser.getReceivedBinlogBytes();
|
||||
holder.isParallel = parser.isParallel();
|
||||
Preconditions.checkNotNull(holder.eventsPublishBlockingTime);
|
||||
Preconditions.checkNotNull(holder.receivedBinlogBytes);
|
||||
return holder;
|
||||
}
|
||||
|
||||
private GroupParserMetricsHolder groupHolder(String destination, GroupEventParser group) {
|
||||
List<CanalEventParser> parsers = group.getEventParsers();
|
||||
GroupParserMetricsHolder groupHolder = new GroupParserMetricsHolder();
|
||||
int num = parsers.size();
|
||||
for (int i = 0; i < num; i ++) {
|
||||
CanalEventParser parser = parsers.get(i);
|
||||
if (parser instanceof AbstractMysqlEventParser) {
|
||||
ParserMetricsHolder single = singleHolder(destination, (AbstractMysqlEventParser)parser, Integer.toString(i + 1));
|
||||
groupHolder.holders.add(single);
|
||||
} else {
|
||||
logger.warn("Null or non AbstractMysqlEventParser, ignore.");
|
||||
}
|
||||
}
|
||||
return groupHolder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void unregister(CanalInstance instance) {
|
||||
final String destination = instance.getDestination();
|
||||
@@ -106,11 +142,17 @@ public class ParserCollector extends Collector implements InstanceRegistry {
|
||||
}
|
||||
|
||||
private class ParserMetricsHolder {
|
||||
private List<String> destLabelValues;
|
||||
private List<String> parserLabelValues;
|
||||
private List<String> modeLabelValues;
|
||||
// metrics for single parser
|
||||
private AtomicLong receivedBinlogBytes;
|
||||
private AtomicLong eventsPublishBlockingTime;
|
||||
// parser mode
|
||||
private boolean isParallel;
|
||||
}
|
||||
|
||||
private class GroupParserMetricsHolder extends ParserMetricsHolder {
|
||||
private final List<ParserMetricsHolder> holders = new ArrayList<>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+7
-9
@@ -99,15 +99,13 @@ public class StoreCollector extends Collector implements InstanceRegistry {
|
||||
long get = Math.min(smh.getExecTime.get(), pet);
|
||||
long aet = Math.min(smh.ackExecTime.get(), get);
|
||||
long now = System.currentTimeMillis();
|
||||
if (now >= pet) {
|
||||
putDelay.addMetric(smh.destLabelValues, (now - pet));
|
||||
}
|
||||
if (now >= get) {
|
||||
getDelay.addMetric(smh.destLabelValues, (now - get));
|
||||
}
|
||||
if (now >= aet) {
|
||||
ackDelay.addMetric(smh.destLabelValues, (now - aet));
|
||||
}
|
||||
// execTime > now，delay显示为0
|
||||
long pd = (now >= pet) ? (now - pet) : 0;
|
||||
putDelay.addMetric(smh.destLabelValues, pd);
|
||||
long gd = (now >= get) ? (now - get) : 0;
|
||||
getDelay.addMetric(smh.destLabelValues, gd);
|
||||
long ad = (now >= aet) ? (now - aet) : 0;
|
||||
ackDelay.addMetric(smh.destLabelValues, ad);
|
||||
putRows.addMetric(smh.destLabelValues, smh.putTableRows.doubleValue());
|
||||
getRows.addMetric(smh.destLabelValues, smh.getTableRows.doubleValue());
|
||||
ackRows.addMetric(smh.destLabelValues, smh.ackTableRows.doubleValue());
|
||||
|
||||
@@ -1792,7 +1792,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -1812,7 +1812,7 @@ public final class CanalEntry {
|
||||
getLogfileNameBytes() {
|
||||
Object ref = logfileName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
logfileName_ = b;
|
||||
@@ -1892,7 +1892,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -1912,7 +1912,7 @@ public final class CanalEntry {
|
||||
getServerenCodeBytes() {
|
||||
Object ref = serverenCode_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
serverenCode_ = b;
|
||||
@@ -1992,7 +1992,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -2012,7 +2012,7 @@ public final class CanalEntry {
|
||||
getSchemaNameBytes() {
|
||||
Object ref = schemaName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
schemaName_ = b;
|
||||
@@ -2046,7 +2046,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -2066,7 +2066,7 @@ public final class CanalEntry {
|
||||
getTableNameBytes() {
|
||||
Object ref = tableName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
tableName_ = b;
|
||||
@@ -2201,7 +2201,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -2221,7 +2221,7 @@ public final class CanalEntry {
|
||||
getGtidBytes() {
|
||||
Object ref = gtid_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
gtid_ = b;
|
||||
@@ -2671,7 +2671,7 @@ public final class CanalEntry {
|
||||
propsBuilder_ = null;
|
||||
props_ = other.props_;
|
||||
bitField0_ = (bitField0_ & ~0x00000800);
|
||||
propsBuilder_ =
|
||||
propsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getPropsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -2802,7 +2802,7 @@ public final class CanalEntry {
|
||||
getLogfileNameBytes() {
|
||||
Object ref = logfileName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
logfileName_ = b;
|
||||
@@ -2998,7 +2998,7 @@ public final class CanalEntry {
|
||||
getServerenCodeBytes() {
|
||||
Object ref = serverenCode_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
serverenCode_ = b;
|
||||
@@ -3197,7 +3197,7 @@ public final class CanalEntry {
|
||||
getSchemaNameBytes() {
|
||||
Object ref = schemaName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
schemaName_ = b;
|
||||
@@ -3297,7 +3297,7 @@ public final class CanalEntry {
|
||||
getTableNameBytes() {
|
||||
Object ref = tableName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
tableName_ = b;
|
||||
@@ -3808,7 +3808,7 @@ public final class CanalEntry {
|
||||
getGtidBytes() {
|
||||
Object ref = gtid_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
gtid_ = b;
|
||||
@@ -4328,7 +4328,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -4348,7 +4348,7 @@ public final class CanalEntry {
|
||||
getNameBytes() {
|
||||
Object ref = name_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
name_ = b;
|
||||
@@ -4506,7 +4506,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -4526,7 +4526,7 @@ public final class CanalEntry {
|
||||
getValueBytes() {
|
||||
Object ref = value_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
value_ = b;
|
||||
@@ -4583,7 +4583,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -4603,7 +4603,7 @@ public final class CanalEntry {
|
||||
getMysqlTypeBytes() {
|
||||
Object ref = mysqlType_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
mysqlType_ = b;
|
||||
@@ -4990,7 +4990,7 @@ public final class CanalEntry {
|
||||
propsBuilder_ = null;
|
||||
props_ = other.props_;
|
||||
bitField0_ = (bitField0_ & ~0x00000040);
|
||||
propsBuilder_ =
|
||||
propsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getPropsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -5177,7 +5177,7 @@ public final class CanalEntry {
|
||||
getNameBytes() {
|
||||
Object ref = name_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
name_ = b;
|
||||
@@ -5733,7 +5733,7 @@ public final class CanalEntry {
|
||||
getValueBytes() {
|
||||
Object ref = value_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
value_ = b;
|
||||
@@ -5881,7 +5881,7 @@ public final class CanalEntry {
|
||||
getMysqlTypeBytes() {
|
||||
Object ref = mysqlType_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
mysqlType_ = b;
|
||||
@@ -6644,7 +6644,7 @@ public final class CanalEntry {
|
||||
beforeColumnsBuilder_ = null;
|
||||
beforeColumns_ = other.beforeColumns_;
|
||||
bitField0_ = (bitField0_ & ~0x00000001);
|
||||
beforeColumnsBuilder_ =
|
||||
beforeColumnsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getBeforeColumnsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -6670,7 +6670,7 @@ public final class CanalEntry {
|
||||
afterColumnsBuilder_ = null;
|
||||
afterColumns_ = other.afterColumns_;
|
||||
bitField0_ = (bitField0_ & ~0x00000002);
|
||||
afterColumnsBuilder_ =
|
||||
afterColumnsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getAfterColumnsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -6696,7 +6696,7 @@ public final class CanalEntry {
|
||||
propsBuilder_ = null;
|
||||
props_ = other.props_;
|
||||
bitField0_ = (bitField0_ & ~0x00000004);
|
||||
propsBuilder_ =
|
||||
propsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getPropsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -8115,7 +8115,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -8135,7 +8135,7 @@ public final class CanalEntry {
|
||||
getSqlBytes() {
|
||||
Object ref = sql_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
sql_ = b;
|
||||
@@ -8279,7 +8279,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -8299,7 +8299,7 @@ public final class CanalEntry {
|
||||
getDdlSchemaNameBytes() {
|
||||
Object ref = ddlSchemaName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
ddlSchemaName_ = b;
|
||||
@@ -8648,7 +8648,7 @@ public final class CanalEntry {
|
||||
rowDatasBuilder_ = null;
|
||||
rowDatas_ = other.rowDatas_;
|
||||
bitField0_ = (bitField0_ & ~0x00000010);
|
||||
rowDatasBuilder_ =
|
||||
rowDatasBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getRowDatasFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -8674,7 +8674,7 @@ public final class CanalEntry {
|
||||
propsBuilder_ = null;
|
||||
props_ = other.props_;
|
||||
bitField0_ = (bitField0_ & ~0x00000020);
|
||||
propsBuilder_ =
|
||||
propsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getPropsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -8904,7 +8904,7 @@ public final class CanalEntry {
|
||||
getSqlBytes() {
|
||||
Object ref = sql_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
sql_ = b;
|
||||
@@ -9628,7 +9628,7 @@ public final class CanalEntry {
|
||||
getDdlSchemaNameBytes() {
|
||||
Object ref = ddlSchemaName_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
ddlSchemaName_ = b;
|
||||
@@ -9973,7 +9973,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -9993,7 +9993,7 @@ public final class CanalEntry {
|
||||
getTransactionIdBytes() {
|
||||
Object ref = transactionId_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
transactionId_ = b;
|
||||
@@ -10362,7 +10362,7 @@ public final class CanalEntry {
|
||||
propsBuilder_ = null;
|
||||
props_ = other.props_;
|
||||
bitField0_ = (bitField0_ & ~0x00000004);
|
||||
propsBuilder_ =
|
||||
propsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getPropsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -10491,7 +10491,7 @@ public final class CanalEntry {
|
||||
getTransactionIdBytes() {
|
||||
Object ref = transactionId_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
transactionId_ = b;
|
||||
@@ -11174,7 +11174,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -11194,7 +11194,7 @@ public final class CanalEntry {
|
||||
getTransactionIdBytes() {
|
||||
Object ref = transactionId_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
transactionId_ = b;
|
||||
@@ -11526,7 +11526,7 @@ public final class CanalEntry {
|
||||
propsBuilder_ = null;
|
||||
props_ = other.props_;
|
||||
bitField0_ = (bitField0_ & ~0x00000004);
|
||||
propsBuilder_ =
|
||||
propsBuilder_ =
|
||||
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
|
||||
getPropsFieldBuilder() : null;
|
||||
} else {
|
||||
@@ -11652,7 +11652,7 @@ public final class CanalEntry {
|
||||
getTransactionIdBytes() {
|
||||
Object ref = transactionId_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
transactionId_ = b;
|
||||
@@ -12188,7 +12188,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -12204,7 +12204,7 @@ public final class CanalEntry {
|
||||
getKeyBytes() {
|
||||
Object ref = key_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
key_ = b;
|
||||
@@ -12230,7 +12230,7 @@ public final class CanalEntry {
|
||||
if (ref instanceof String) {
|
||||
return (String) ref;
|
||||
} else {
|
||||
com.google.protobuf.ByteString bs =
|
||||
com.google.protobuf.ByteString bs =
|
||||
(com.google.protobuf.ByteString) ref;
|
||||
String s = bs.toStringUtf8();
|
||||
if (bs.isValidUtf8()) {
|
||||
@@ -12246,7 +12246,7 @@ public final class CanalEntry {
|
||||
getValueBytes() {
|
||||
Object ref = value_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
value_ = b;
|
||||
@@ -12541,7 +12541,7 @@ public final class CanalEntry {
|
||||
getKeyBytes() {
|
||||
Object ref = key_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
key_ = b;
|
||||
@@ -12617,7 +12617,7 @@ public final class CanalEntry {
|
||||
getValueBytes() {
|
||||
Object ref = value_;
|
||||
if (ref instanceof String) {
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString b =
|
||||
com.google.protobuf.ByteString.copyFromUtf8(
|
||||
(String) ref);
|
||||
value_ = b;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,4 @@
|
||||
syntax = "proto3";
|
||||
package com.alibaba.otter.canal.protocol;
|
||||
|
||||
option java_package = "com.alibaba.otter.canal.protocol";
|
||||
@@ -5,6 +6,7 @@ option java_outer_classname = "CanalPacket";
|
||||
option optimize_for = SPEED;
|
||||
|
||||
enum Compression {
|
||||
COMPRESSIONCOMPATIBLEPROTO2 = 0;
|
||||
NONE = 1;
|
||||
ZLIB = 2;
|
||||
GZIP = 3;
|
||||
@@ -12,6 +14,8 @@ enum Compression {
|
||||
}
|
||||
|
||||
enum PacketType {
|
||||
//compatible
|
||||
PACKAGETYPECOMPATIBLEPROTO2 = 0;
|
||||
HANDSHAKE = 1;
|
||||
CLIENTAUTHENTICATION = 2;
|
||||
ACK = 3;
|
||||
@@ -29,86 +33,122 @@ enum PacketType {
|
||||
}
|
||||
|
||||
message Packet {
|
||||
optional int32 magic_number = 1 [default = 17];
|
||||
optional int32 version = 2 [default = 1];
|
||||
optional PacketType type = 3;
|
||||
optional Compression compression = 4 [default = NONE];
|
||||
optional bytes body = 5;
|
||||
//[default = 17];
|
||||
oneof magic_number_present {
|
||||
int32 magic_number = 1;
|
||||
}
|
||||
//[default = 1];
|
||||
oneof version_present {
|
||||
int32 version = 2;
|
||||
};
|
||||
PacketType type = 3;
|
||||
//[default = NONE];
|
||||
oneof compression_present {
|
||||
Compression compression = 4;
|
||||
}
|
||||
|
||||
bytes body = 5;
|
||||
}
|
||||
|
||||
message HeartBeat {
|
||||
optional int64 send_timestamp = 1;
|
||||
optional int64 start_timestamp = 2;
|
||||
int64 send_timestamp = 1;
|
||||
int64 start_timestamp = 2;
|
||||
}
|
||||
|
||||
message Handshake {
|
||||
optional string communication_encoding = 1 [default = "utf8"];
|
||||
optional bytes seeds = 2;
|
||||
repeated Compression supported_compressions = 3;
|
||||
// [default = "utf8"];
|
||||
oneof communication_encoding_present {
|
||||
string communication_encoding = 1;
|
||||
}
|
||||
bytes seeds = 2;
|
||||
Compression supported_compressions = 3;
|
||||
}
|
||||
|
||||
// client authentication
|
||||
message ClientAuth {
|
||||
optional string username = 1;
|
||||
optional bytes password = 2; // hashed password with seeds from Handshake message
|
||||
optional int32 net_read_timeout = 3 [default = 0]; // in seconds
|
||||
optional int32 net_write_timeout = 4 [default = 0]; // in seconds
|
||||
optional string destination = 5;
|
||||
optional string client_id = 6;
|
||||
optional string filter = 7;
|
||||
optional int64 start_timestamp = 8;
|
||||
string username = 1;
|
||||
bytes password = 2; // hashed password with seeds from Handshake message
|
||||
// [default = 0]
|
||||
oneof net_read_timeout_present {
|
||||
int32 net_read_timeout = 3; // in seconds
|
||||
}
|
||||
// [default = 0];
|
||||
oneof net_write_timeout_present {
|
||||
int32 net_write_timeout = 4; // in seconds
|
||||
}
|
||||
string destination = 5;
|
||||
string client_id = 6;
|
||||
string filter = 7;
|
||||
int64 start_timestamp = 8;
|
||||
}
|
||||
|
||||
message Ack {
|
||||
optional int32 error_code = 1 [default = 0];
|
||||
optional string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
|
||||
//[default = 0]
|
||||
oneof error_code_present {
|
||||
int32 error_code = 1;
|
||||
}
|
||||
string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
|
||||
}
|
||||
|
||||
message ClientAck {
|
||||
optional string destination = 1;
|
||||
optional string client_id = 2;
|
||||
optional int64 batch_id = 3;
|
||||
string destination = 1;
|
||||
string client_id = 2;
|
||||
int64 batch_id = 3;
|
||||
}
|
||||
|
||||
// subscription
|
||||
message Sub {
|
||||
optional string destination = 1;
|
||||
optional string client_id = 2;
|
||||
optional string filter = 7;
|
||||
string destination = 1;
|
||||
string client_id = 2;
|
||||
string filter = 7;
|
||||
}
|
||||
|
||||
// Unsubscription
|
||||
message Unsub {
|
||||
optional string destination = 1;
|
||||
optional string client_id = 2;
|
||||
optional string filter = 7;
|
||||
string destination = 1;
|
||||
string client_id = 2;
|
||||
string filter = 7;
|
||||
}
|
||||
|
||||
// PullRequest
|
||||
message Get {
|
||||
optional string destination = 1;
|
||||
optional string client_id = 2;
|
||||
optional int32 fetch_size = 3;
|
||||
optional int64 timeout = 4 [default = -1]; // 默认-1时代表不控制
|
||||
optional int32 unit = 5 [default = 2];// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
|
||||
optional bool auto_ack = 6 [default = false]; // 是否自动ack
|
||||
string destination = 1;
|
||||
string client_id = 2;
|
||||
int32 fetch_size = 3;
|
||||
//[default = -1]
|
||||
oneof timeout_present {
|
||||
int64 timeout = 4; // 默认-1时代表不控制
|
||||
}
|
||||
//[default = 2]
|
||||
oneof unit_present {
|
||||
int32 unit = 5;// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
|
||||
}
|
||||
//[default = false]
|
||||
oneof auto_ack_present {
|
||||
bool auto_ack = 6; // 是否自动ack
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
message Messages {
|
||||
optional int64 batch_id = 1;
|
||||
int64 batch_id = 1;
|
||||
repeated bytes messages = 2;
|
||||
}
|
||||
|
||||
// TBD when new packets are required
|
||||
message Dump{
|
||||
optional string journal = 1;
|
||||
optional int64 position = 2;
|
||||
optional int64 timestamp = 3 [default = 0];
|
||||
string journal = 1;
|
||||
int64 position = 2;
|
||||
// [default = 0]
|
||||
oneof timestamp_present {
|
||||
int64 timestamp = 3;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
message ClientRollback{
|
||||
optional string destination = 1;
|
||||
optional string client_id = 2;
|
||||
optional int64 batch_id = 3;
|
||||
string destination = 1;
|
||||
string client_id = 2;
|
||||
int64 batch_id = 3;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
syntax = "proto3";
|
||||
package com.alibaba.otter.canal.protocol;
|
||||
|
||||
option java_package = "com.alibaba.otter.canal.protocol";
|
||||
@@ -10,88 +11,101 @@ option optimize_for = SPEED;
|
||||
****************************************************************/
|
||||
message Entry {
|
||||
/**协议头部信息**/
|
||||
optional Header header = 1;
|
||||
|
||||
/**打散后的事件类型**/
|
||||
optional EntryType entryType = 2 [default = ROWDATA];
|
||||
|
||||
Header header = 1;
|
||||
///**打散后的事件类型**/ [default = ROWDATA]
|
||||
oneof entryType_present{
|
||||
EntryType entryType = 2;
|
||||
}
|
||||
|
||||
/**传输的二进制数组**/
|
||||
optional bytes storeValue = 3;
|
||||
bytes storeValue = 3;
|
||||
}
|
||||
|
||||
/**message Header**/
|
||||
message Header {
|
||||
/**协议的版本号**/
|
||||
optional int32 version = 1 [default = 1];
|
||||
|
||||
/**协议的版本号**/ //[default = 1]
|
||||
oneof version_present {
|
||||
int32 version = 1;
|
||||
}
|
||||
|
||||
|
||||
/**binlog/redolog 文件名**/
|
||||
optional string logfileName = 2;
|
||||
|
||||
string logfileName = 2;
|
||||
|
||||
/**binlog/redolog 文件的偏移位置**/
|
||||
optional int64 logfileOffset = 3;
|
||||
|
||||
int64 logfileOffset = 3;
|
||||
|
||||
/**服务端serverId**/
|
||||
optional int64 serverId = 4;
|
||||
|
||||
int64 serverId = 4;
|
||||
|
||||
/** 变更数据的编码 **/
|
||||
optional string serverenCode = 5;
|
||||
|
||||
string serverenCode = 5;
|
||||
|
||||
/**变更数据的执行时间 **/
|
||||
optional int64 executeTime = 6;
|
||||
|
||||
/** 变更数据的来源**/
|
||||
optional Type sourceType = 7 [default = MYSQL];
|
||||
|
||||
int64 executeTime = 6;
|
||||
|
||||
/** 变更数据的来源**/ //[default = MYSQL]
|
||||
oneof sourceType_present {
|
||||
Type sourceType = 7;
|
||||
}
|
||||
|
||||
|
||||
/** 变更数据的schemaname**/
|
||||
optional string schemaName = 8;
|
||||
|
||||
string schemaName = 8;
|
||||
|
||||
/**变更数据的tablename**/
|
||||
optional string tableName = 9;
|
||||
|
||||
string tableName = 9;
|
||||
|
||||
/**每个event的长度**/
|
||||
optional int64 eventLength = 10;
|
||||
|
||||
/**数据变更类型**/
|
||||
optional EventType eventType = 11 [default = UPDATE];
|
||||
|
||||
int64 eventLength = 10;
|
||||
|
||||
/**数据变更类型**/ // [default = UPDATE]
|
||||
oneof eventType_present {
|
||||
EventType eventType = 11;
|
||||
}
|
||||
|
||||
|
||||
/**预留扩展**/
|
||||
repeated Pair props = 12;
|
||||
|
||||
/**当前事务的gitd**/
|
||||
optional string gtid = 13;
|
||||
string gtid = 13;
|
||||
}
|
||||
|
||||
/**每个字段的数据结构**/
|
||||
message Column {
|
||||
/**字段下标**/
|
||||
optional int32 index = 1;
|
||||
|
||||
int32 index = 1;
|
||||
|
||||
/**字段java中类型**/
|
||||
optional int32 sqlType = 2;
|
||||
|
||||
int32 sqlType = 2;
|
||||
|
||||
/**字段名称(忽略大小写)，在mysql中是没有的**/
|
||||
optional string name = 3;
|
||||
|
||||
string name = 3;
|
||||
|
||||
/**是否是主键**/
|
||||
optional bool isKey = 4;
|
||||
|
||||
bool isKey = 4;
|
||||
|
||||
/**如果EventType=UPDATE,用于标识这个字段值是否有修改**/
|
||||
optional bool updated = 5;
|
||||
|
||||
/** 标识是否为空 **/
|
||||
optional bool isNull = 6 [default = false];
|
||||
|
||||
bool updated = 5;
|
||||
|
||||
/** 标识是否为空 **/ //[default = false]
|
||||
oneof isNull_present {
|
||||
bool isNull = 6;
|
||||
}
|
||||
|
||||
|
||||
/**预留扩展**/
|
||||
repeated Pair props = 7;
|
||||
|
||||
repeated Pair props = 7;
|
||||
|
||||
/** 字段值,timestamp,Datetime是一个时间格式的文本 **/
|
||||
optional string value = 8;
|
||||
|
||||
string value = 8;
|
||||
|
||||
/** 对应数据对象原始长度 **/
|
||||
optional int32 length = 9;
|
||||
|
||||
int32 length = 9;
|
||||
|
||||
/**字段mysql类型**/
|
||||
optional string mysqlType = 10;
|
||||
string mysqlType = 10;
|
||||
}
|
||||
|
||||
message RowData {
|
||||
@@ -101,73 +115,81 @@ message RowData {
|
||||
|
||||
/** 字段信息，增量数据(修改后,新增后) **/
|
||||
repeated Column afterColumns = 2;
|
||||
|
||||
|
||||
/**预留扩展**/
|
||||
repeated Pair props = 3;
|
||||
repeated Pair props = 3;
|
||||
}
|
||||
|
||||
/**message row 每行变更数据的数据结构**/
|
||||
message RowChange {
|
||||
|
||||
/**tableId,由数据库产生**/
|
||||
optional int64 tableId = 1;
|
||||
|
||||
/**数据变更类型**/
|
||||
optional EventType eventType = 2 [default = UPDATE];
|
||||
|
||||
/** 标识是否是ddl语句 **/
|
||||
optional bool isDdl = 10 [default = false];
|
||||
int64 tableId = 1;
|
||||
|
||||
|
||||
/**数据变更类型**/ //[default = UPDATE]
|
||||
oneof eventType_present {
|
||||
EventType eventType = 2;
|
||||
}
|
||||
|
||||
|
||||
/** 标识是否是ddl语句 **/ // [default = false]
|
||||
oneof isDdl_present {
|
||||
bool isDdl = 10;
|
||||
}
|
||||
|
||||
|
||||
/** ddl/query的sql语句 **/
|
||||
optional string sql = 11;
|
||||
|
||||
string sql = 11;
|
||||
|
||||
/** 一次数据库变更可能存在多行 **/
|
||||
repeated RowData rowDatas = 12;
|
||||
|
||||
|
||||
/**预留扩展**/
|
||||
repeated Pair props = 13;
|
||||
|
||||
repeated Pair props = 13;
|
||||
|
||||
/** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName **/
|
||||
optional string ddlSchemaName = 14;
|
||||
string ddlSchemaName = 14;
|
||||
}
|
||||
|
||||
/**开始事务的一些信息**/
|
||||
message TransactionBegin{
|
||||
|
||||
|
||||
/**已废弃，请使用header里的executeTime**/
|
||||
optional int64 executeTime = 1;
|
||||
|
||||
int64 executeTime = 1;
|
||||
|
||||
/**已废弃，Begin里不提供事务id**/
|
||||
optional string transactionId = 2;
|
||||
|
||||
string transactionId = 2;
|
||||
|
||||
/**预留扩展**/
|
||||
repeated Pair props = 3;
|
||||
|
||||
repeated Pair props = 3;
|
||||
|
||||
/**执行的thread Id**/
|
||||
optional int64 threadId = 4;
|
||||
int64 threadId = 4;
|
||||
}
|
||||
|
||||
/**结束事务的一些信息**/
|
||||
message TransactionEnd{
|
||||
|
||||
|
||||
/**已废弃，请使用header里的executeTime**/
|
||||
optional int64 executeTime = 1;
|
||||
|
||||
int64 executeTime = 1;
|
||||
|
||||
/**事务号**/
|
||||
optional string transactionId = 2;
|
||||
|
||||
string transactionId = 2;
|
||||
|
||||
/**预留扩展**/
|
||||
repeated Pair props = 3;
|
||||
repeated Pair props = 3;
|
||||
}
|
||||
|
||||
/**预留扩展**/
|
||||
message Pair{
|
||||
optional string key = 1;
|
||||
optional string value = 2;
|
||||
string key = 1;
|
||||
string value = 2;
|
||||
}
|
||||
|
||||
/**打散后的事件类型，主要用于标识事务的开始，变更数据，结束**/
|
||||
enum EntryType{
|
||||
ENTRYTYPECOMPATIBLEPROTO2 = 0;
|
||||
TRANSACTIONBEGIN = 1;
|
||||
ROWDATA = 2;
|
||||
TRANSACTIONEND = 3;
|
||||
@@ -178,6 +200,7 @@ enum EntryType{
|
||||
|
||||
/** 事件类型 **/
|
||||
enum EventType {
|
||||
EVENTTYPECOMPATIBLEPROTO2 = 0;
|
||||
INSERT = 1;
|
||||
UPDATE = 2;
|
||||
DELETE = 3;
|
||||
@@ -200,6 +223,7 @@ enum EventType {
|
||||
|
||||
/**数据库类型**/
|
||||
enum Type {
|
||||
TYPECOMPATIBLEPROTO2 = 0;
|
||||
ORACLE = 1;
|
||||
MYSQL = 2;
|
||||
PGSQL = 3;
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
package com.alibaba.otter.canal.protocol;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.*;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
|
||||
/**
|
||||
* @author machengyuan 2018-9-13 下午10:31:14
|
||||
* @version 1.0.0
|
||||
*/
|
||||
public class FlatMessage implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = -3386650678735860050L;
|
||||
|
||||
private long id;
|
||||
private String database;
|
||||
private String table;
|
||||
private Boolean isDdl;
|
||||
private String type;
|
||||
private Long ts;
|
||||
private String sql;
|
||||
private Map<String, Integer> sqlType;
|
||||
private Map<String, String> mysqlType;
|
||||
private List<Map<String, String>> data;
|
||||
private List<Map<String, String>> old;
|
||||
|
||||
public FlatMessage(){
|
||||
}
|
||||
|
||||
public FlatMessage(long id){
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public long getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(long id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getDatabase() {
|
||||
return database;
|
||||
}
|
||||
|
||||
public void setDatabase(String database) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
public String getTable() {
|
||||
return table;
|
||||
}
|
||||
|
||||
public void setTable(String table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
public Boolean getIsDdl() {
|
||||
return isDdl;
|
||||
}
|
||||
|
||||
public void setIsDdl(Boolean isDdl) {
|
||||
this.isDdl = isDdl;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public Long getTs() {
|
||||
return ts;
|
||||
}
|
||||
|
||||
public void setTs(Long ts) {
|
||||
this.ts = ts;
|
||||
}
|
||||
|
||||
public String getSql() {
|
||||
return sql;
|
||||
}
|
||||
|
||||
public void setSql(String sql) {
|
||||
this.sql = sql;
|
||||
}
|
||||
|
||||
public Map<String, Integer> getSqlType() {
|
||||
return sqlType;
|
||||
}
|
||||
|
||||
public void setSqlType(Map<String, Integer> sqlType) {
|
||||
this.sqlType = sqlType;
|
||||
}
|
||||
|
||||
public Map<String, String> getMysqlType() {
|
||||
return mysqlType;
|
||||
}
|
||||
|
||||
public void setMysqlType(Map<String, String> mysqlType) {
|
||||
this.mysqlType = mysqlType;
|
||||
}
|
||||
|
||||
public List<Map<String, String>> getData() {
|
||||
return data;
|
||||
}
|
||||
|
||||
public void setData(List<Map<String, String>> data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public List<Map<String, String>> getOld() {
|
||||
return old;
|
||||
}
|
||||
|
||||
public void setOld(List<Map<String, String>> old) {
|
||||
this.old = old;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将Message转换为FlatMessage
|
||||
*
|
||||
* @param message 原生message
|
||||
* @return FlatMessage列表
|
||||
*/
|
||||
public static List<FlatMessage> messageConverter(Message message) {
|
||||
try {
|
||||
if (message == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<FlatMessage> flatMessages = new ArrayList<>();
|
||||
|
||||
List<ByteString> rawEntries = message.getRawEntries();
|
||||
|
||||
for (ByteString byteString : rawEntries) {
|
||||
CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
|
||||
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|
||||
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
|
||||
continue;
|
||||
}
|
||||
|
||||
CanalEntry.RowChange rowChange;
|
||||
try {
|
||||
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(
|
||||
"ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
|
||||
e);
|
||||
}
|
||||
|
||||
CanalEntry.EventType eventType = rowChange.getEventType();
|
||||
|
||||
FlatMessage flatMessage = new FlatMessage(message.getId());
|
||||
flatMessages.add(flatMessage);
|
||||
flatMessage.setDatabase(entry.getHeader().getSchemaName());
|
||||
flatMessage.setTable(entry.getHeader().getTableName());
|
||||
flatMessage.setIsDdl(rowChange.getIsDdl());
|
||||
flatMessage.setType(eventType.toString());
|
||||
flatMessage.setTs(System.currentTimeMillis());
|
||||
flatMessage.setSql(rowChange.getSql());
|
||||
|
||||
if (!rowChange.getIsDdl()) {
|
||||
Map<String, Integer> sqlType = new LinkedHashMap<>();
|
||||
Map<String, String> mysqlType = new LinkedHashMap<>();
|
||||
List<Map<String, String>> data = new ArrayList<>();
|
||||
List<Map<String, String>> old = new ArrayList<>();
|
||||
|
||||
Set<String> updateSet = new HashSet<>();
|
||||
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
|
||||
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
|
||||
&& eventType != CanalEntry.EventType.DELETE) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Map<String, String> row = new LinkedHashMap<>();
|
||||
List<CanalEntry.Column> columns;
|
||||
|
||||
if (eventType == CanalEntry.EventType.DELETE) {
|
||||
columns = rowData.getBeforeColumnsList();
|
||||
} else {
|
||||
columns = rowData.getAfterColumnsList();
|
||||
}
|
||||
|
||||
for (CanalEntry.Column column : columns) {
|
||||
sqlType.put(column.getName(), column.getSqlType());
|
||||
mysqlType.put(column.getName(), column.getMysqlType());
|
||||
row.put(column.getName(), column.getValue());
|
||||
// 获取update为true的字段
|
||||
if (column.getUpdated()) {
|
||||
updateSet.add(column.getName());
|
||||
}
|
||||
}
|
||||
if (!row.isEmpty()) {
|
||||
data.add(row);
|
||||
}
|
||||
|
||||
if (eventType == CanalEntry.EventType.UPDATE) {
|
||||
Map<String, String> rowOld = new LinkedHashMap<>();
|
||||
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
|
||||
if (updateSet.contains(column.getName())) {
|
||||
rowOld.put(column.getName(), column.getValue());
|
||||
}
|
||||
}
|
||||
// update操作将记录修改前的值
|
||||
if (!rowOld.isEmpty()) {
|
||||
old.add(rowOld);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!sqlType.isEmpty()) {
|
||||
flatMessage.setSqlType(sqlType);
|
||||
}
|
||||
if (!mysqlType.isEmpty()) {
|
||||
flatMessage.setMysqlType(mysqlType);
|
||||
}
|
||||
if (!data.isEmpty()) {
|
||||
flatMessage.setData(data);
|
||||
}
|
||||
if (!old.isEmpty()) {
|
||||
flatMessage.setOld(old);
|
||||
}
|
||||
}
|
||||
}
|
||||
return flatMessages;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将FlatMessage按指定的字段值hash拆分
|
||||
*
|
||||
* @param flatMessage flatMessage
|
||||
* @param partitionsNum 分区数量
|
||||
* @param pkHashConfig hash映射
|
||||
* @return 拆分后的flatMessage数组
|
||||
*/
|
||||
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum,
|
||||
Map<String, String> pkHashConfig) {
|
||||
if (partitionsNum == null) {
|
||||
partitionsNum = 1;
|
||||
}
|
||||
FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
|
||||
|
||||
String pk = pkHashConfig.get(flatMessage.getDatabase() + "." + flatMessage.getTable());
|
||||
if (pk == null || flatMessage.getIsDdl()) {
|
||||
partitionMessages[0] = flatMessage;
|
||||
} else {
|
||||
if (flatMessage.getData() != null) {
|
||||
int idx = 0;
|
||||
for (Map<String, String> row : flatMessage.getData()) {
|
||||
String value = row.get(pk);
|
||||
if (value == null) {
|
||||
value = "";
|
||||
}
|
||||
int hash = value.hashCode();
|
||||
int pkHash = Math.abs(hash) % partitionsNum;
|
||||
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
|
||||
pkHash = Math.abs(pkHash);
|
||||
|
||||
FlatMessage flatMessageTmp = partitionMessages[pkHash];
|
||||
if (flatMessageTmp == null) {
|
||||
flatMessageTmp = new FlatMessage(flatMessage.getId());
|
||||
partitionMessages[pkHash] = flatMessageTmp;
|
||||
flatMessageTmp.setDatabase(flatMessage.getDatabase());
|
||||
flatMessageTmp.setTable(flatMessage.getTable());
|
||||
flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
|
||||
flatMessageTmp.setType(flatMessage.getType());
|
||||
flatMessageTmp.setSql(flatMessage.getSql());
|
||||
flatMessageTmp.setSqlType(flatMessage.getSqlType());
|
||||
flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
|
||||
}
|
||||
List<Map<String, String>> data = flatMessageTmp.getData();
|
||||
if (data == null) {
|
||||
data = new ArrayList<>();
|
||||
flatMessageTmp.setData(data);
|
||||
}
|
||||
data.add(row);
|
||||
if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
|
||||
List<Map<String, String>> old = flatMessageTmp.getOld();
|
||||
if (old == null) {
|
||||
old = new ArrayList<>();
|
||||
flatMessageTmp.setOld(old);
|
||||
}
|
||||
old.add(flatMessage.getOld().get(idx));
|
||||
}
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return partitionMessages;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "FlatMessage{" + "id=" + id + ", database='" + database + '\'' + ", table='" + table + '\'' + ", isDdl="
|
||||
+ isDdl + ", type='" + type + '\'' + ", ts=" + ts + ", sql='" + sql + '\'' + ", sqlType=" + sqlType
|
||||
+ ", mysqlType=" + mysqlType + ", data=" + data + ", old=" + old + '}';
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.common;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.CodedOutputStream;
|
||||
@@ -31,14 +32,14 @@ public class CanalMessageSerializer {
|
||||
// packet size
|
||||
int size = 0;
|
||||
size += CodedOutputStream.computeEnumSize(3,
|
||||
CanalPacket.PacketType.MESSAGES.getNumber());
|
||||
PacketType.MESSAGES.getNumber());
|
||||
size += CodedOutputStream.computeTagSize(5)
|
||||
+ CodedOutputStream.computeRawVarint32Size(messageSize)
|
||||
+ messageSize;
|
||||
// build data
|
||||
byte[] body = new byte[size];
|
||||
CodedOutputStream output = CodedOutputStream.newInstance(body);
|
||||
output.writeEnum(3, CanalPacket.PacketType.MESSAGES.getNumber());
|
||||
output.writeEnum(3, PacketType.MESSAGES.getNumber());
|
||||
|
||||
output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
|
||||
output.writeRawVarint32(messageSize);
|
||||
@@ -56,7 +57,7 @@ public class CanalMessageSerializer {
|
||||
}
|
||||
|
||||
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
|
||||
packetBuilder.setType(CanalPacket.PacketType.MESSAGES);
|
||||
packetBuilder.setType(PacketType.MESSAGES);
|
||||
packetBuilder.setBody(messageBuilder.build().toByteString());
|
||||
return packetBuilder.build().toByteArray();
|
||||
}
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
package com.alibaba.otter.canal.common;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* kafka 配置项
|
||||
@@ -13,23 +12,26 @@ import java.util.Set;
|
||||
*/
|
||||
public class MQProperties {
|
||||
|
||||
private String servers = "localhost:6667";
|
||||
private int retries = 0;
|
||||
private int batchSize = 16384;
|
||||
private int lingerMs = 1;
|
||||
private long bufferMemory = 33554432L;
|
||||
private boolean filterTransactionEntry = true;
|
||||
private int canalBatchSize = 5;
|
||||
private String producerGroup = "Canal-Producer";
|
||||
private String servers = "localhost:6667";
|
||||
private int retries = 0;
|
||||
private int batchSize = 16384;
|
||||
private int lingerMs = 1;
|
||||
private long bufferMemory = 33554432L;
|
||||
private boolean filterTransactionEntry = true;
|
||||
private String producerGroup = "Canal-Producer";
|
||||
private int canalBatchSize = 50;
|
||||
private Long canalGetTimeout;
|
||||
private boolean flatMessage = true;
|
||||
|
||||
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
|
||||
private List<CanalDestination> canalDestinations = new ArrayList<CanalDestination>();
|
||||
|
||||
public static class CanalDestination {
|
||||
|
||||
private String canalDestination;
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
private Set<Topic> topics = new HashSet<Topic>();
|
||||
private String canalDestination;
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
private Integer partitionsNum;
|
||||
private Map<String, String> partitionHash;
|
||||
|
||||
public String getCanalDestination() {
|
||||
return canalDestination;
|
||||
@@ -55,54 +57,23 @@ public class MQProperties {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
public Set<Topic> getTopics() {
|
||||
return topics;
|
||||
public Integer getPartitionsNum() {
|
||||
return partitionsNum;
|
||||
}
|
||||
|
||||
public void setTopics(Set<Topic> topics) {
|
||||
this.topics = topics;
|
||||
public void setPartitionsNum(Integer partitionsNum) {
|
||||
this.partitionsNum = partitionsNum;
|
||||
}
|
||||
|
||||
public Map<String, String> getPartitionHash() {
|
||||
return partitionHash;
|
||||
}
|
||||
|
||||
public void setPartitionHash(Map<String, String> partitionHash) {
|
||||
this.partitionHash = partitionHash;
|
||||
}
|
||||
}
|
||||
|
||||
public static class Topic {
|
||||
|
||||
private String topic;
|
||||
private Integer partition;
|
||||
|
||||
public String getTopic() {
|
||||
return topic;
|
||||
}
|
||||
|
||||
public void setTopic(String topic) {
|
||||
this.topic = topic;
|
||||
}
|
||||
|
||||
public Integer getPartition() {
|
||||
return partition;
|
||||
}
|
||||
|
||||
public void setPartition(Integer partition) {
|
||||
this.partition = partition;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
Topic topic1 = (Topic) o;
|
||||
|
||||
if (topic != null ? !topic.equals(topic1.topic) : topic1.topic != null) return false;
|
||||
return partition != null ? partition.equals(topic1.partition) : topic1.partition == null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
int result = topic != null ? topic.hashCode() : 0;
|
||||
result = 31 * result + (partition != null ? partition.hashCode() : 0);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
public String getServers() {
|
||||
return servers;
|
||||
@@ -152,6 +123,22 @@ public class MQProperties {
|
||||
this.canalBatchSize = canalBatchSize;
|
||||
}
|
||||
|
||||
public Long getCanalGetTimeout() {
|
||||
return canalGetTimeout;
|
||||
}
|
||||
|
||||
public void setCanalGetTimeout(Long canalGetTimeout) {
|
||||
this.canalGetTimeout = canalGetTimeout;
|
||||
}
|
||||
|
||||
public boolean getFlatMessage() {
|
||||
return flatMessage;
|
||||
}
|
||||
|
||||
public void setFlatMessage(boolean flatMessage) {
|
||||
this.flatMessage = flatMessage;
|
||||
}
|
||||
|
||||
public List<CanalDestination> getCanalDestinations() {
|
||||
return canalDestinations;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package com.alibaba.otter.canal.kafka;
|
||||
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.protocol.FlatMessage;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.spi.CanalMQProducer;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.Producer;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
@@ -12,8 +14,6 @@ import org.apache.kafka.common.serialization.StringSerializer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
|
||||
/**
|
||||
* kafka producer 主操作类
|
||||
*
|
||||
@@ -26,8 +26,13 @@ public class CanalKafkaProducer implements CanalMQProducer {
|
||||
|
||||
private Producer<String, Message> producer;
|
||||
|
||||
private Producer<String, String> producer2; // 用于扁平message的数据投递
|
||||
|
||||
private MQProperties kafkaProperties;
|
||||
|
||||
@Override
|
||||
public void init(MQProperties kafkaProperties) {
|
||||
this.kafkaProperties = kafkaProperties;
|
||||
Properties properties = new Properties();
|
||||
properties.put("bootstrap.servers", kafkaProperties.getServers());
|
||||
properties.put("acks", "all");
|
||||
@@ -36,15 +41,27 @@ public class CanalKafkaProducer implements CanalMQProducer {
|
||||
properties.put("linger.ms", kafkaProperties.getLingerMs());
|
||||
properties.put("buffer.memory", kafkaProperties.getBufferMemory());
|
||||
properties.put("key.serializer", StringSerializer.class.getName());
|
||||
properties.put("value.serializer", MessageSerializer.class.getName());
|
||||
producer = new KafkaProducer<String, Message>(properties);
|
||||
if (!kafkaProperties.getFlatMessage()) {
|
||||
properties.put("value.serializer", MessageSerializer.class.getName());
|
||||
producer = new KafkaProducer<String, Message>(properties);
|
||||
} else {
|
||||
properties.put("value.serializer", StringSerializer.class.getName());
|
||||
producer2 = new KafkaProducer<String, String>(properties);
|
||||
}
|
||||
|
||||
// producer.initTransactions();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
try {
|
||||
logger.info("## stop the kafka producer");
|
||||
producer.close();
|
||||
if (producer != null) {
|
||||
producer.close();
|
||||
}
|
||||
if (producer2 != null) {
|
||||
producer2.close();
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.warn("##something goes wrong when stopping kafka producer:", e);
|
||||
} finally {
|
||||
@@ -53,16 +70,72 @@ public class CanalKafkaProducer implements CanalMQProducer {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void send(MQProperties.Topic topic, Message message) throws IOException {
|
||||
ProducerRecord<String, Message> record;
|
||||
if (topic.getPartition() != null) {
|
||||
record = new ProducerRecord<String, Message>(topic.getTopic(), topic.getPartition(), null, message);
|
||||
} else {
|
||||
record = new ProducerRecord<String, Message>(topic.getTopic(), message);
|
||||
}
|
||||
producer.send(record);
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message to kafka topic: {} \n {}", topic.getTopic(), message.toString());
|
||||
public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
|
||||
try {
|
||||
// producer.beginTransaction();
|
||||
if (!kafkaProperties.getFlatMessage()) {
|
||||
ProducerRecord<String, Message> record;
|
||||
if (canalDestination.getPartition() != null) {
|
||||
record = new ProducerRecord<String, Message>(canalDestination.getTopic(),
|
||||
canalDestination.getPartition(),
|
||||
null,
|
||||
message);
|
||||
} else {
|
||||
record = new ProducerRecord<String, Message>(canalDestination.getTopic(), 0, null, message);
|
||||
}
|
||||
|
||||
producer.send(record);
|
||||
} else {
|
||||
// 发送扁平数据json
|
||||
List<FlatMessage> flatMessages = FlatMessage.messageConverter(message);
|
||||
if (flatMessages != null) {
|
||||
for (FlatMessage flatMessage : flatMessages) {
|
||||
if (canalDestination.getPartition() != null) {
|
||||
ProducerRecord<String, String> record = new ProducerRecord<String, String>(canalDestination
|
||||
.getTopic(), canalDestination.getPartition(), null, JSON.toJSONString(flatMessage));
|
||||
producer2.send(record);
|
||||
} else {
|
||||
if (canalDestination.getPartitionHash() != null
|
||||
&& !canalDestination.getPartitionHash().isEmpty()) {
|
||||
FlatMessage[] partitionFlatMessage = FlatMessage.messagePartition(flatMessage,
|
||||
canalDestination.getPartitionsNum(),
|
||||
canalDestination.getPartitionHash());
|
||||
int length = partitionFlatMessage.length;
|
||||
for (int i = 0; i < length; i++) {
|
||||
FlatMessage flatMessagePart = partitionFlatMessage[i];
|
||||
if (flatMessagePart != null) {
|
||||
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
|
||||
canalDestination.getTopic(),
|
||||
i,
|
||||
null,
|
||||
JSON.toJSONString(flatMessagePart));
|
||||
producer2.send(record);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ProducerRecord<String, String> record = new ProducerRecord<String, String>(
|
||||
canalDestination.getTopic(),
|
||||
0,
|
||||
null,
|
||||
JSON.toJSONString(flatMessage));
|
||||
producer2.send(record);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// producer.commitTransaction();
|
||||
callback.commit();
|
||||
if (logger.isDebugEnabled()) {
|
||||
logger.debug("send message to kafka topic: {}", canalDestination.getTopic());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
// producer.abortTransaction();
|
||||
callback.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,20 +1,9 @@
|
||||
package com.alibaba.otter.canal.kafka;
|
||||
|
||||
import com.alibaba.otter.canal.common.CanalMessageSerializer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.kafka.common.errors.SerializationException;
|
||||
import org.apache.kafka.common.serialization.Serializer;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalEntry;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket;
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.CodedOutputStream;
|
||||
import com.google.protobuf.WireFormat;
|
||||
import java.util.Map;
|
||||
import org.apache.kafka.common.serialization.Serializer;
|
||||
|
||||
/**
|
||||
* Kafka Message类的序列化
|
||||
|
||||
@@ -35,17 +35,24 @@ public class CanalRocketMQProducer implements CanalMQProducer {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void send(final MQProperties.Topic topic, com.alibaba.otter.canal.protocol.Message data) {
|
||||
public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
|
||||
Callback callback) {
|
||||
try {
|
||||
Message message = new Message(topic.getTopic(), CanalMessageSerializer.serializer(data));
|
||||
Message message = new Message(destination.getTopic(), CanalMessageSerializer.serializer(data));
|
||||
this.defaultMQProducer.send(message, new MessageQueueSelector() {
|
||||
@Override
|
||||
public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
|
||||
return mqs.get(topic.getPartition());
|
||||
int partition = 0;
|
||||
if (destination.getPartition() != null) {
|
||||
partition = destination.getPartition();
|
||||
}
|
||||
return mqs.get(partition);
|
||||
}
|
||||
}, null);
|
||||
callback.commit();
|
||||
} catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
|
||||
logger.error("Send message error!", e);
|
||||
callback.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.alibaba.otter.canal.server;
|
||||
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
|
||||
import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
|
||||
@@ -9,17 +10,18 @@ import java.io.FileInputStream;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
public class CanalMQStarter {
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
|
||||
private static final Logger logger = LoggerFactory.getLogger(CanalMQStarter.class);
|
||||
|
||||
private static final String CLASSPATH_URL_PREFIX = "classpath:";
|
||||
|
||||
private volatile boolean running = false;
|
||||
private volatile boolean running = false;
|
||||
|
||||
private ExecutorService executorService;
|
||||
|
||||
@@ -27,7 +29,7 @@ public class CanalMQStarter {
|
||||
|
||||
private MQProperties properties;
|
||||
|
||||
public CanalMQStarter(CanalMQProducer canalMQProducer){
|
||||
public CanalMQStarter(CanalMQProducer canalMQProducer) {
|
||||
this.canalMQProducer = canalMQProducer;
|
||||
}
|
||||
|
||||
@@ -51,6 +53,7 @@ public class CanalMQStarter {
|
||||
if (properties.isFilterTransactionEntry()) {
|
||||
System.setProperty("canal.instance.filter.transaction.entry", "true");
|
||||
}
|
||||
|
||||
// 对应每个instance启动一个worker线程
|
||||
List<MQProperties.CanalDestination> destinations = properties.getCanalDestinations();
|
||||
|
||||
@@ -95,8 +98,8 @@ public class CanalMQStarter {
|
||||
while (!running)
|
||||
;
|
||||
logger.info("## start the canal consumer: {}.", destination.getCanalDestination());
|
||||
CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
|
||||
ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
|
||||
final CanalServerWithEmbedded server = CanalServerWithEmbedded.instance();
|
||||
final ClientIdentity clientIdentity = new ClientIdentity(destination.getCanalDestination(), (short) 1001, "");
|
||||
while (running) {
|
||||
try {
|
||||
if (!server.getCanalInstances().containsKey(clientIdentity.getDestination())) {
|
||||
@@ -111,27 +114,41 @@ public class CanalMQStarter {
|
||||
logger.info("## the canal consumer {} is running now ......", destination.getCanalDestination());
|
||||
|
||||
while (running) {
|
||||
Message message = server.getWithoutAck(clientIdentity, properties.getCanalBatchSize()); // 获取指定数量的数据
|
||||
long batchId = message.getId();
|
||||
Message message;
|
||||
if (properties.getCanalGetTimeout() != null) {
|
||||
message = server.getWithoutAck(clientIdentity,
|
||||
properties.getCanalBatchSize(),
|
||||
properties.getCanalGetTimeout(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
} else {
|
||||
message = server.getWithoutAck(clientIdentity, properties.getCanalBatchSize());
|
||||
}
|
||||
|
||||
final long batchId = message.getId();
|
||||
try {
|
||||
int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
|
||||
if (batchId != -1 && size != 0) {
|
||||
if (!StringUtils.isEmpty(destination.getTopic())) {
|
||||
MQProperties.Topic topic = new MQProperties.Topic();
|
||||
topic.setTopic(destination.getTopic());
|
||||
topic.setPartition(destination.getPartition());
|
||||
destination.getTopics().add(topic);
|
||||
}
|
||||
for (MQProperties.Topic topic : destination.getTopics()) {
|
||||
canalMQProducer.send(topic, message); // 发送message到所有topic
|
||||
canalMQProducer.send(destination, message, new CanalKafkaProducer.Callback() {
|
||||
|
||||
@Override
|
||||
public void commit() {
|
||||
server.ack(clientIdentity, batchId); // 提交确认
|
||||
}
|
||||
|
||||
@Override
|
||||
public void rollback() {
|
||||
server.rollback(clientIdentity, batchId);
|
||||
}
|
||||
}); // 发送message到topic
|
||||
} else {
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
if (batchId != -1) {
|
||||
server.ack(clientIdentity, batchId); // 提交确认
|
||||
}
|
||||
} catch (Exception e) {
|
||||
server.rollback(clientIdentity);
|
||||
logger.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
+4
-3
@@ -311,9 +311,10 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
|
||||
}
|
||||
|
||||
if (CollectionUtils.isEmpty(events.getEvents())) {
|
||||
logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result is null",
|
||||
clientIdentity.getClientId(),
|
||||
batchSize);
|
||||
// logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result
|
||||
// is null",
|
||||
// clientIdentity.getClientId(),
|
||||
// batchSize);
|
||||
return new Message(-1, true, new ArrayList()); // 返回空包，避免生成batchId，浪费性能
|
||||
} else {
|
||||
// 记录到流式信息
|
||||
|
||||
+2
-1
@@ -3,7 +3,6 @@ package com.alibaba.otter.canal.server.netty.handler;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.exception.ExceptionUtils;
|
||||
import org.jboss.netty.buffer.ChannelBuffer;
|
||||
@@ -34,6 +33,7 @@ import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
|
||||
import com.alibaba.otter.canal.server.netty.NettyUtils;
|
||||
import com.alibaba.otter.canal.server.netty.listener.ChannelFutureAggregator;
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.protobuf.CodedOutputStream;
|
||||
import com.google.protobuf.WireFormat;
|
||||
@@ -50,6 +50,7 @@ public class SessionHandler extends SimpleChannelHandler {
|
||||
this.embeddedServer = embeddedServer;
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
|
||||
logger.info("message receives in session handler...");
|
||||
long start = System.nanoTime();
|
||||
|
||||
+9
-9
@@ -2,7 +2,7 @@ package com.alibaba.otter.canal.server.netty.listener;
|
||||
|
||||
import com.alibaba.otter.canal.protocol.CanalPacket;
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.protobuf.GeneratedMessage;
|
||||
import com.google.protobuf.GeneratedMessageV3;
|
||||
import org.jboss.netty.channel.ChannelFuture;
|
||||
import org.jboss.netty.channel.ChannelFutureListener;
|
||||
|
||||
@@ -16,19 +16,19 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
|
||||
|
||||
private ClientRequestResult result;
|
||||
|
||||
public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, boolean empty) {
|
||||
public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, boolean empty) {
|
||||
this(destination, request, type, amount, latency, empty, (short) 0);
|
||||
}
|
||||
|
||||
public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency) {
|
||||
public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency) {
|
||||
this(destination, request, type, amount, latency, false, (short) 0);
|
||||
}
|
||||
|
||||
public ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, short errorCode) {
|
||||
public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, short errorCode) {
|
||||
this(destination, request, type, amount, latency, false, errorCode);
|
||||
}
|
||||
|
||||
private ChannelFutureAggregator(String destination, GeneratedMessage request, CanalPacket.PacketType type, int amount, long latency, boolean empty, short errorCode) {
|
||||
private ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, boolean empty, short errorCode) {
|
||||
this.result = new ClientRequestResult.Builder()
|
||||
.destination(destination)
|
||||
.type(type)
|
||||
@@ -56,7 +56,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
|
||||
|
||||
private String destination;
|
||||
private CanalPacket.PacketType type;
|
||||
private GeneratedMessage request;
|
||||
private GeneratedMessageV3 request;
|
||||
private int amount;
|
||||
private long latency;
|
||||
private short errorCode;
|
||||
@@ -81,7 +81,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
|
||||
|
||||
private String destination;
|
||||
private CanalPacket.PacketType type;
|
||||
private GeneratedMessage request;
|
||||
private GeneratedMessageV3 request;
|
||||
private int amount;
|
||||
private long latency;
|
||||
private short errorCode;
|
||||
@@ -98,7 +98,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
|
||||
return this;
|
||||
}
|
||||
|
||||
Builder request(GeneratedMessage request) {
|
||||
Builder request(GeneratedMessageV3 request) {
|
||||
this.request = request;
|
||||
return this;
|
||||
}
|
||||
@@ -153,7 +153,7 @@ public class ChannelFutureAggregator implements ChannelFutureListener {
|
||||
return type;
|
||||
}
|
||||
|
||||
public GeneratedMessage getRequest() {
|
||||
public GeneratedMessageV3 getRequest() {
|
||||
return request;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.alibaba.otter.canal.spi;
|
||||
|
||||
import com.alibaba.otter.canal.common.MQProperties;
|
||||
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
|
||||
import com.alibaba.otter.canal.protocol.Message;
|
||||
import java.io.IOException;
|
||||
|
||||
@@ -15,14 +16,21 @@ public interface CanalMQProducer {
|
||||
/**
|
||||
* Send canal message to related topic
|
||||
*
|
||||
* @param topic MQ topic
|
||||
* @param canalDestination canal mq destination
|
||||
* @param message canal message
|
||||
* @throws IOException
|
||||
*/
|
||||
void send(MQProperties.Topic topic, Message message) throws IOException;
|
||||
void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException;
|
||||
|
||||
/**
|
||||
* Stop MQ producer service
|
||||
*/
|
||||
void stop();
|
||||
|
||||
interface Callback {
|
||||
|
||||
void commit();
|
||||
|
||||
void rollback();
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,8 @@ public class CanalServerWithNettyTest {
|
||||
|
||||
@Test
|
||||
public void testAuth() {
|
||||
|
||||
|
||||
try {
|
||||
SocketChannel channel = SocketChannel.open();
|
||||
channel.connect(new InetSocketAddress("127.0.0.1", 1088));
|
||||
@@ -80,7 +82,7 @@ public class CanalServerWithNettyTest {
|
||||
}
|
||||
//
|
||||
Handshake handshake = Handshake.parseFrom(p.getBody());
|
||||
System.out.println(handshake.getSupportedCompressionsList());
|
||||
System.out.println(handshake.getSupportedCompressions());
|
||||
//
|
||||
ClientAuth ca = ClientAuth.newBuilder()
|
||||
.setUsername("")
|
||||
|
||||
@@ -33,10 +33,13 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
|
||||
private static final Logger logger = LoggerFactory.getLogger(EntryEventSink.class);
|
||||
private static final int maxFullTimes = 10;
|
||||
private CanalEventStore<Event> eventStore;
|
||||
protected boolean filterTransactionEntry = false; // 是否需要过滤事务头/尾
|
||||
protected boolean filterTransactionEntry = false; // 是否需要尽可能过滤事务头/尾
|
||||
protected boolean filterEmtryTransactionEntry = true; // 是否需要过滤空的事务头/尾
|
||||
protected long emptyTransactionInterval = 5 * 1000; // 空的事务输出的频率
|
||||
protected long emptyTransctionThresold = 8192; // 超过1024个事务头，输出一个
|
||||
protected long emptyTransctionThresold = 8192; // 超过8192个事务头，输出一个
|
||||
|
||||
protected volatile long lastTransactionTimestamp = 0L;
|
||||
protected AtomicLong lastTransactionCount = new AtomicLong(0L);
|
||||
protected volatile long lastEmptyTransactionTimestamp = 0L;
|
||||
protected AtomicLong lastEmptyTransactionCount = new AtomicLong(0L);
|
||||
private AtomicLong eventsSinkBlockingTime = new AtomicLong(0L);
|
||||
@@ -74,17 +77,7 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
|
||||
public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination)
|
||||
throws CanalSinkException,
|
||||
InterruptedException {
|
||||
List rowDatas = entrys;
|
||||
if (filterTransactionEntry) {
|
||||
rowDatas = new ArrayList<CanalEntry.Entry>();
|
||||
for (CanalEntry.Entry entry : entrys) {
|
||||
if (entry.getEntryType() == EntryType.ROWDATA) {
|
||||
rowDatas.add(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sinkData(rowDatas, remoteAddress);
|
||||
return sinkData(entrys, remoteAddress);
|
||||
}
|
||||
|
||||
private boolean sinkData(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress)
|
||||
@@ -97,17 +90,26 @@ public class EntryEventSink extends AbstractCanalEventSink<List<CanalEntry.Entry
|
||||
continue;
|
||||
}
|
||||
|
||||
Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
|
||||
events.add(event);
|
||||
if (filterTransactionEntry
|
||||
&& (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND)) {
|
||||
long currentTimestamp = entry.getHeader().getExecuteTime();
|
||||
// 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常
|
||||
if (Math.abs(currentTimestamp - lastTransactionTimestamp) > emptyTransactionInterval
|
||||
|| lastTransactionCount.incrementAndGet() > emptyTransctionThresold) {
|
||||
lastTransactionCount.set(0L);
|
||||
lastTransactionTimestamp = currentTimestamp;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
hasRowData |= (entry.getEntryType() == EntryType.ROWDATA);
|
||||
hasHeartBeat |= (entry.getEntryType() == EntryType.HEARTBEAT);
|
||||
Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
|
||||
events.add(event);
|
||||
}
|
||||
|
||||
if (hasRowData) {
|
||||
// 存在row记录
|
||||
return doSink(events);
|
||||
} else if (hasHeartBeat) {
|
||||
// 存在heartbeat记录，直接跳给后续处理
|
||||
if (hasRowData || hasHeartBeat) {
|
||||
// 存在row记录 或者 存在heartbeat记录，直接跳给后续处理
|
||||
return doSink(events);
|
||||
} else {
|
||||
// 需要过滤的数据
|
||||
|
||||
+13
-13
@@ -35,16 +35,16 @@ import com.alibaba.otter.canal.store.model.Events;
|
||||
*/
|
||||
public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge implements CanalEventStore<Event>, CanalStoreScavenge {
|
||||
|
||||
private static final long INIT_SQEUENCE = -1;
|
||||
private static final long INIT_SEQUENCE = -1;
|
||||
private int bufferSize = 16 * 1024;
|
||||
private int bufferMemUnit = 1024; // memsize的单位，默认为1kb大小
|
||||
private int indexMask;
|
||||
private Event[] entries;
|
||||
|
||||
// 记录下put/get/ack操作的三个下标
|
||||
private AtomicLong putSequence = new AtomicLong(INIT_SQEUENCE); // 代表当前put操作最后一次写操作发生的位置
|
||||
private AtomicLong getSequence = new AtomicLong(INIT_SQEUENCE); // 代表当前get操作读取的最后一条的位置
|
||||
private AtomicLong ackSequence = new AtomicLong(INIT_SQEUENCE); // 代表当前ack操作的最后一条的位置
|
||||
private AtomicLong putSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前put操作最后一次写操作发生的位置
|
||||
private AtomicLong getSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前get操作读取的最后一条的位置
|
||||
private AtomicLong ackSequence = new AtomicLong(INIT_SEQUENCE); // 代表当前ack操作的最后一条的位置
|
||||
|
||||
// 记录下put/get/ack操作的三个memsize大小
|
||||
private AtomicLong putMemSize = new AtomicLong(0);
|
||||
@@ -358,17 +358,17 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
|
||||
lock.lock();
|
||||
try {
|
||||
long firstSeqeuence = ackSequence.get();
|
||||
if (firstSeqeuence == INIT_SQEUENCE && firstSeqeuence < putSequence.get()) {
|
||||
if (firstSeqeuence == INIT_SEQUENCE && firstSeqeuence < putSequence.get()) {
|
||||
// 没有ack过数据
|
||||
Event event = entries[getIndex(firstSeqeuence + 1)]; // 最后一次ack为-1，需要移动到下一条,included
|
||||
// = false
|
||||
return CanalEventUtils.createPosition(event, false);
|
||||
} else if (firstSeqeuence > INIT_SQEUENCE && firstSeqeuence < putSequence.get()) {
|
||||
} else if (firstSeqeuence > INIT_SEQUENCE && firstSeqeuence < putSequence.get()) {
|
||||
// ack未追上put操作
|
||||
Event event = entries[getIndex(firstSeqeuence + 1)]; // 最后一次ack的位置数据
|
||||
// + 1
|
||||
return CanalEventUtils.createPosition(event, true);
|
||||
} else if (firstSeqeuence > INIT_SQEUENCE && firstSeqeuence == putSequence.get()) {
|
||||
} else if (firstSeqeuence > INIT_SEQUENCE && firstSeqeuence == putSequence.get()) {
|
||||
// 已经追上，store中没有数据
|
||||
Event event = entries[getIndex(firstSeqeuence)]; // 最后一次ack的位置数据，和last为同一条，included
|
||||
// = false
|
||||
@@ -387,10 +387,10 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
|
||||
lock.lock();
|
||||
try {
|
||||
long latestSequence = putSequence.get();
|
||||
if (latestSequence > INIT_SQEUENCE && latestSequence != ackSequence.get()) {
|
||||
if (latestSequence > INIT_SEQUENCE && latestSequence != ackSequence.get()) {
|
||||
Event event = entries[(int) putSequence.get() & indexMask]; // 最后一次写入的数据，最后一条未消费的数据
|
||||
return CanalEventUtils.createPosition(event, true);
|
||||
} else if (latestSequence > INIT_SQEUENCE && latestSequence == ackSequence.get()) {
|
||||
} else if (latestSequence > INIT_SEQUENCE && latestSequence == ackSequence.get()) {
|
||||
// ack已经追上了put操作
|
||||
Event event = entries[(int) putSequence.get() & indexMask]; // 最后一次写入的数据，included
|
||||
// =
|
||||
@@ -435,7 +435,7 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
|
||||
if (batchMode.isMemSize()) {
|
||||
ackMemSize.addAndGet(memsize);
|
||||
// 尝试清空buffer中的内存，将ack之前的内存全部释放掉
|
||||
for (long index = sequence + 1; index <= next; index++) {
|
||||
for (long index = sequence + 1; index < next; index++) {
|
||||
entries[getIndex(index)] = null;// 设置为null
|
||||
}
|
||||
}
|
||||
@@ -473,9 +473,9 @@ public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge imple
|
||||
final ReentrantLock lock = this.lock;
|
||||
lock.lock();
|
||||
try {
|
||||
putSequence.set(INIT_SQEUENCE);
|
||||
getSequence.set(INIT_SQEUENCE);
|
||||
ackSequence.set(INIT_SQEUENCE);
|
||||
putSequence.set(INIT_SEQUENCE);
|
||||
getSequence.set(INIT_SEQUENCE);
|
||||
ackSequence.set(INIT_SEQUENCE);
|
||||
|
||||
putMemSize.set(0);
|
||||
getMemSize.set(0);
|
||||
|
||||
Reference in New Issue
Block a user