wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge canal repo master

Browse Source
This commit is contained in:
duhengforever
2018-09-27 17:09:00 +08:00
parent a37a7ff142 ff592758cf
commit 5ae16b1c7c
73 changed files with 12936 additions and 10729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
+51 -51
View File
@@ -1792,7 +1792,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -1812,7 +1812,7 @@ public final class CanalEntry {
getLogfileNameBytes() {
Object ref = logfileName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
logfileName_ = b;
@@ -1892,7 +1892,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -1912,7 +1912,7 @@ public final class CanalEntry {
getServerenCodeBytes() {
Object ref = serverenCode_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
serverenCode_ = b;
@@ -1992,7 +1992,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -2012,7 +2012,7 @@ public final class CanalEntry {
getSchemaNameBytes() {
Object ref = schemaName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
schemaName_ = b;
@@ -2046,7 +2046,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -2066,7 +2066,7 @@ public final class CanalEntry {
getTableNameBytes() {
Object ref = tableName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
tableName_ = b;
@@ -2201,7 +2201,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -2221,7 +2221,7 @@ public final class CanalEntry {
getGtidBytes() {
Object ref = gtid_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
gtid_ = b;
@@ -2671,7 +2671,7 @@ public final class CanalEntry {
propsBuilder_ = null;
props_ = other.props_;
bitField0_ = (bitField0_ & ~0x00000800);
propsBuilder_ =
propsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getPropsFieldBuilder() : null;
} else {
@@ -2802,7 +2802,7 @@ public final class CanalEntry {
getLogfileNameBytes() {
Object ref = logfileName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
logfileName_ = b;
@@ -2998,7 +2998,7 @@ public final class CanalEntry {
getServerenCodeBytes() {
Object ref = serverenCode_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
serverenCode_ = b;
@@ -3197,7 +3197,7 @@ public final class CanalEntry {
getSchemaNameBytes() {
Object ref = schemaName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
schemaName_ = b;
@@ -3297,7 +3297,7 @@ public final class CanalEntry {
getTableNameBytes() {
Object ref = tableName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
tableName_ = b;
@@ -3808,7 +3808,7 @@ public final class CanalEntry {
getGtidBytes() {
Object ref = gtid_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
gtid_ = b;
@@ -4328,7 +4328,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -4348,7 +4348,7 @@ public final class CanalEntry {
getNameBytes() {
Object ref = name_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
name_ = b;
@@ -4506,7 +4506,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -4526,7 +4526,7 @@ public final class CanalEntry {
getValueBytes() {
Object ref = value_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
value_ = b;
@@ -4583,7 +4583,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -4603,7 +4603,7 @@ public final class CanalEntry {
getMysqlTypeBytes() {
Object ref = mysqlType_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
mysqlType_ = b;
@@ -4990,7 +4990,7 @@ public final class CanalEntry {
propsBuilder_ = null;
props_ = other.props_;
bitField0_ = (bitField0_ & ~0x00000040);
propsBuilder_ =
propsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getPropsFieldBuilder() : null;
} else {
@@ -5177,7 +5177,7 @@ public final class CanalEntry {
getNameBytes() {
Object ref = name_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
name_ = b;
@@ -5733,7 +5733,7 @@ public final class CanalEntry {
getValueBytes() {
Object ref = value_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
value_ = b;
@@ -5881,7 +5881,7 @@ public final class CanalEntry {
getMysqlTypeBytes() {
Object ref = mysqlType_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
mysqlType_ = b;
@@ -6644,7 +6644,7 @@ public final class CanalEntry {
beforeColumnsBuilder_ = null;
beforeColumns_ = other.beforeColumns_;
bitField0_ = (bitField0_ & ~0x00000001);
beforeColumnsBuilder_ =
beforeColumnsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getBeforeColumnsFieldBuilder() : null;
} else {
@@ -6670,7 +6670,7 @@ public final class CanalEntry {
afterColumnsBuilder_ = null;
afterColumns_ = other.afterColumns_;
bitField0_ = (bitField0_ & ~0x00000002);
afterColumnsBuilder_ =
afterColumnsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getAfterColumnsFieldBuilder() : null;
} else {
@@ -6696,7 +6696,7 @@ public final class CanalEntry {
propsBuilder_ = null;
props_ = other.props_;
bitField0_ = (bitField0_ & ~0x00000004);
propsBuilder_ =
propsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getPropsFieldBuilder() : null;
} else {
@@ -8115,7 +8115,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -8135,7 +8135,7 @@ public final class CanalEntry {
getSqlBytes() {
Object ref = sql_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
sql_ = b;
@@ -8279,7 +8279,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -8299,7 +8299,7 @@ public final class CanalEntry {
getDdlSchemaNameBytes() {
Object ref = ddlSchemaName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
ddlSchemaName_ = b;
@@ -8648,7 +8648,7 @@ public final class CanalEntry {
rowDatasBuilder_ = null;
rowDatas_ = other.rowDatas_;
bitField0_ = (bitField0_ & ~0x00000010);
rowDatasBuilder_ =
rowDatasBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getRowDatasFieldBuilder() : null;
} else {
@@ -8674,7 +8674,7 @@ public final class CanalEntry {
propsBuilder_ = null;
props_ = other.props_;
bitField0_ = (bitField0_ & ~0x00000020);
propsBuilder_ =
propsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getPropsFieldBuilder() : null;
} else {
@@ -8904,7 +8904,7 @@ public final class CanalEntry {
getSqlBytes() {
Object ref = sql_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
sql_ = b;
@@ -9628,7 +9628,7 @@ public final class CanalEntry {
getDdlSchemaNameBytes() {
Object ref = ddlSchemaName_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
ddlSchemaName_ = b;
@@ -9973,7 +9973,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -9993,7 +9993,7 @@ public final class CanalEntry {
getTransactionIdBytes() {
Object ref = transactionId_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
transactionId_ = b;
@@ -10362,7 +10362,7 @@ public final class CanalEntry {
propsBuilder_ = null;
props_ = other.props_;
bitField0_ = (bitField0_ & ~0x00000004);
propsBuilder_ =
propsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getPropsFieldBuilder() : null;
} else {
@@ -10491,7 +10491,7 @@ public final class CanalEntry {
getTransactionIdBytes() {
Object ref = transactionId_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
transactionId_ = b;
@@ -11174,7 +11174,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -11194,7 +11194,7 @@ public final class CanalEntry {
getTransactionIdBytes() {
Object ref = transactionId_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
transactionId_ = b;
@@ -11526,7 +11526,7 @@ public final class CanalEntry {
propsBuilder_ = null;
props_ = other.props_;
bitField0_ = (bitField0_ & ~0x00000004);
propsBuilder_ =
propsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getPropsFieldBuilder() : null;
} else {
@@ -11652,7 +11652,7 @@ public final class CanalEntry {
getTransactionIdBytes() {
Object ref = transactionId_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
transactionId_ = b;
@@ -12188,7 +12188,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -12204,7 +12204,7 @@ public final class CanalEntry {
getKeyBytes() {
Object ref = key_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
key_ = b;
@@ -12230,7 +12230,7 @@ public final class CanalEntry {
if (ref instanceof String) {
return (String) ref;
} else {
com.google.protobuf.ByteString bs =
com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (bs.isValidUtf8()) {
@@ -12246,7 +12246,7 @@ public final class CanalEntry {
getValueBytes() {
Object ref = value_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
value_ = b;
@@ -12541,7 +12541,7 @@ public final class CanalEntry {
getKeyBytes() {
Object ref = key_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
key_ = b;
@@ -12617,7 +12617,7 @@ public final class CanalEntry {
getValueBytes() {
Object ref = value_;
if (ref instanceof String) {
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8(
(String) ref);
value_ = b;
protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java
+10718 -9575
View File
File diff suppressed because it is too large Load Diff
protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalProtocol.proto
+82 -42
View File
@@ -1,3 +1,4 @@
syntax = "proto3";
package com.alibaba.otter.canal.protocol;
option java_package = "com.alibaba.otter.canal.protocol";
@@ -5,6 +6,7 @@ option java_outer_classname = "CanalPacket";
option optimize_for = SPEED;
enum Compression {
COMPRESSIONCOMPATIBLEPROTO2 = 0;
NONE = 1;
ZLIB = 2;
GZIP = 3;
@@ -12,6 +14,8 @@ enum Compression {
}
enum PacketType {
//compatible
PACKAGETYPECOMPATIBLEPROTO2 = 0;
HANDSHAKE = 1;
CLIENTAUTHENTICATION = 2;
ACK = 3;
@@ -29,86 +33,122 @@ enum PacketType {
}
message Packet {
optional int32 magic_number = 1 [default = 17];
optional int32 version = 2 [default = 1];
optional PacketType type = 3;
optional Compression compression = 4 [default = NONE];
optional bytes body = 5;
//[default = 17];
oneof magic_number_present {
int32 magic_number = 1;
}
//[default = 1];
oneof version_present {
int32 version = 2;
};
PacketType type = 3;
//[default = NONE];
oneof compression_present {
Compression compression = 4;
}
bytes body = 5;
}
message HeartBeat {
optional int64 send_timestamp = 1;
optional int64 start_timestamp = 2;
int64 send_timestamp = 1;
int64 start_timestamp = 2;
}
message Handshake {
optional string communication_encoding = 1 [default = "utf8"];
optional bytes seeds = 2;
repeated Compression supported_compressions = 3;
// [default = "utf8"];
oneof communication_encoding_present {
string communication_encoding = 1;
}
bytes seeds = 2;
Compression supported_compressions = 3;
}
// client authentication
message ClientAuth {
optional string username = 1;
optional bytes password = 2; // hashed password with seeds from Handshake message
optional int32 net_read_timeout = 3 [default = 0]; // in seconds
optional int32 net_write_timeout = 4 [default = 0]; // in seconds
optional string destination = 5;
optional string client_id = 6;
optional string filter = 7;
optional int64 start_timestamp = 8;
string username = 1;
bytes password = 2; // hashed password with seeds from Handshake message
// [default = 0]
oneof net_read_timeout_present {
int32 net_read_timeout = 3; // in seconds
}
// [default = 0];
oneof net_write_timeout_present {
int32 net_write_timeout = 4; // in seconds
}
string destination = 5;
string client_id = 6;
string filter = 7;
int64 start_timestamp = 8;
}
message Ack {
optional int32 error_code = 1 [default = 0];
optional string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
//[default = 0]
oneof error_code_present {
int32 error_code = 1;
}
string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
}
message ClientAck {
optional string destination = 1;
optional string client_id = 2;
optional int64 batch_id = 3;
string destination = 1;
string client_id = 2;
int64 batch_id = 3;
}
// subscription
message Sub {
optional string destination = 1;
optional string client_id = 2;
optional string filter = 7;
string destination = 1;
string client_id = 2;
string filter = 7;
}
// Unsubscription
message Unsub {
optional string destination = 1;
optional string client_id = 2;
optional string filter = 7;
string destination = 1;
string client_id = 2;
string filter = 7;
}
// PullRequest
message Get {
optional string destination = 1;
optional string client_id = 2;
optional int32 fetch_size = 3;
optional int64 timeout = 4 [default = -1]; // 默认-1时代表不控制
optional int32 unit = 5 [default = 2];// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
optional bool auto_ack = 6 [default = false]; // 是否自动ack
string destination = 1;
string client_id = 2;
int32 fetch_size = 3;
//[default = -1]
oneof timeout_present {
int64 timeout = 4; // 默认-1时代表不控制
}
//[default = 2]
oneof unit_present {
int32 unit = 5;// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
}
//[default = false]
oneof auto_ack_present {
bool auto_ack = 6; // 是否自动ack
}
}
//
message Messages {
optional int64 batch_id = 1;
int64 batch_id = 1;
repeated bytes messages = 2;
}
// TBD when new packets are required
message Dump{
optional string journal = 1;
optional int64 position = 2;
optional int64 timestamp = 3 [default = 0];
string journal = 1;
int64 position = 2;
// [default = 0]
oneof timestamp_present {
int64 timestamp = 3;
}
}
message ClientRollback{
optional string destination = 1;
optional string client_id = 2;
optional int64 batch_id = 3;
string destination = 1;
string client_id = 2;
int64 batch_id = 3;
}
protocol/src/main/java/com/alibaba/otter/canal/protocol/EntryProtocol.proto
+107 -83
View File
@@ -1,3 +1,4 @@
syntax = "proto3";
package com.alibaba.otter.canal.protocol;
option java_package = "com.alibaba.otter.canal.protocol";
@@ -10,88 +11,101 @@ option optimize_for = SPEED;
****************************************************************/
message Entry {
/**协议头部信息**/
optional Header header = 1;
/**打散后的事件类型**/
optional EntryType entryType = 2 [default = ROWDATA];
Header header = 1;
///**打散后的事件类型**/ [default = ROWDATA]
oneof entryType_present{
EntryType entryType = 2;
}
/**传输的二进制数组**/
optional bytes storeValue = 3;
bytes storeValue = 3;
}
/**message Header**/
message Header {
/**协议的版本号**/
optional int32 version = 1 [default = 1];
/**协议的版本号**/ //[default = 1]
oneof version_present {
int32 version = 1;
}
/**binlog/redolog 文件名**/
optional string logfileName = 2;
string logfileName = 2;
/**binlog/redolog 文件的偏移位置**/
optional int64 logfileOffset = 3;
int64 logfileOffset = 3;
/**服务端serverId**/
optional int64 serverId = 4;
int64 serverId = 4;
/** 变更数据的编码 **/
optional string serverenCode = 5;
string serverenCode = 5;
/**变更数据的执行时间 **/
optional int64 executeTime = 6;
/** 变更数据的来源**/
optional Type sourceType = 7 [default = MYSQL];
int64 executeTime = 6;
/** 变更数据的来源**/ //[default = MYSQL]
oneof sourceType_present {
Type sourceType = 7;
}
/** 变更数据的schemaname**/
optional string schemaName = 8;
string schemaName = 8;
/**变更数据的tablename**/
optional string tableName = 9;
string tableName = 9;
/**每个event的长度**/
optional int64 eventLength = 10;
/**数据变更类型**/
optional EventType eventType = 11 [default = UPDATE];
int64 eventLength = 10;
/**数据变更类型**/ // [default = UPDATE]
oneof eventType_present {
EventType eventType = 11;
}
/**预留扩展**/
repeated Pair props = 12;
/**当前事务的gitd**/
optional string gtid = 13;
string gtid = 13;
}
/**每个字段的数据结构**/
message Column {
/**字段下标**/
optional int32 index = 1;
int32 index = 1;
/**字段java中类型**/
optional int32 sqlType = 2;
int32 sqlType = 2;
/**字段名称(忽略大小写)，在mysql中是没有的**/
optional string name = 3;
string name = 3;
/**是否是主键**/
optional bool isKey = 4;
bool isKey = 4;
/**如果EventType=UPDATE,用于标识这个字段值是否有修改**/
optional bool updated = 5;
/** 标识是否为空 **/
optional bool isNull = 6 [default = false];
bool updated = 5;
/** 标识是否为空 **/ //[default = false]
oneof isNull_present {
bool isNull = 6;
}
/**预留扩展**/
repeated Pair props = 7;
repeated Pair props = 7;
/** 字段值,timestamp,Datetime是一个时间格式的文本 **/
optional string value = 8;
string value = 8;
/** 对应数据对象原始长度 **/
optional int32 length = 9;
int32 length = 9;
/**字段mysql类型**/
optional string mysqlType = 10;
string mysqlType = 10;
}
message RowData {
@@ -101,73 +115,81 @@ message RowData {
/** 字段信息，增量数据(修改后,新增后) **/
repeated Column afterColumns = 2;
/**预留扩展**/
repeated Pair props = 3;
repeated Pair props = 3;
}
/**message row 每行变更数据的数据结构**/
message RowChange {
/**tableId,由数据库产生**/
optional int64 tableId = 1;
/**数据变更类型**/
optional EventType eventType = 2 [default = UPDATE];
/** 标识是否是ddl语句 **/
optional bool isDdl = 10 [default = false];
int64 tableId = 1;
/**数据变更类型**/ //[default = UPDATE]
oneof eventType_present {
EventType eventType = 2;
}
/** 标识是否是ddl语句 **/ // [default = false]
oneof isDdl_present {
bool isDdl = 10;
}
/** ddl/query的sql语句 **/
optional string sql = 11;
string sql = 11;
/** 一次数据库变更可能存在多行 **/
repeated RowData rowDatas = 12;
/**预留扩展**/
repeated Pair props = 13;
repeated Pair props = 13;
/** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName **/
optional string ddlSchemaName = 14;
string ddlSchemaName = 14;
}
/**开始事务的一些信息**/
message TransactionBegin{
/**已废弃，请使用header里的executeTime**/
optional int64 executeTime = 1;
int64 executeTime = 1;
/**已废弃，Begin里不提供事务id**/
optional string transactionId = 2;
string transactionId = 2;
/**预留扩展**/
repeated Pair props = 3;
repeated Pair props = 3;
/**执行的thread Id**/
optional int64 threadId = 4;
int64 threadId = 4;
}
/**结束事务的一些信息**/
message TransactionEnd{
/**已废弃，请使用header里的executeTime**/
optional int64 executeTime = 1;
int64 executeTime = 1;
/**事务号**/
optional string transactionId = 2;
string transactionId = 2;
/**预留扩展**/
repeated Pair props = 3;
repeated Pair props = 3;
}
/**预留扩展**/
message Pair{
optional string key = 1;
optional string value = 2;
string key = 1;
string value = 2;
}
/**打散后的事件类型，主要用于标识事务的开始，变更数据，结束**/
enum EntryType{
ENTRYTYPECOMPATIBLEPROTO2 = 0;
TRANSACTIONBEGIN = 1;
ROWDATA = 2;
TRANSACTIONEND = 3;
@@ -178,6 +200,7 @@ enum EntryType{
/** 事件类型 **/
enum EventType {
EVENTTYPECOMPATIBLEPROTO2 = 0;
INSERT = 1;
UPDATE = 2;
DELETE = 3;
@@ -200,6 +223,7 @@ enum EventType {
/**数据库类型**/
enum Type {
TYPECOMPATIBLEPROTO2 = 0;
ORACLE = 1;
MYSQL = 2;
PGSQL = 3;
protocol/src/main/java/com/alibaba/otter/canal/protocol/FlatMessage.java
+304
View File
@@ -0,0 +1,304 @@
package com.alibaba.otter.canal.protocol;
import java.io.Serializable;
import java.util.*;
import com.google.protobuf.ByteString;
/**
* @author machengyuan 2018-9-13 下午10:31:14
* @version 1.0.0
*/
public class FlatMessage implements Serializable {
private static final long serialVersionUID = -3386650678735860050L;
private long id;
private String database;
private String table;
private Boolean isDdl;
private String type;
private Long ts;
private String sql;
private Map<String, Integer> sqlType;
private Map<String, String> mysqlType;
private List<Map<String, String>> data;
private List<Map<String, String>> old;
public FlatMessage(){
}
public FlatMessage(long id){
this.id = id;
}
public long getId() {
return id;
}
public void setId(long id) {
this.id = id;
}
public String getDatabase() {
return database;
}
public void setDatabase(String database) {
this.database = database;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public Boolean getIsDdl() {
return isDdl;
}
public void setIsDdl(Boolean isDdl) {
this.isDdl = isDdl;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public Long getTs() {
return ts;
}
public void setTs(Long ts) {
this.ts = ts;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public Map<String, Integer> getSqlType() {
return sqlType;
}
public void setSqlType(Map<String, Integer> sqlType) {
this.sqlType = sqlType;
}
public Map<String, String> getMysqlType() {
return mysqlType;
}
public void setMysqlType(Map<String, String> mysqlType) {
this.mysqlType = mysqlType;
}
public List<Map<String, String>> getData() {
return data;
}
public void setData(List<Map<String, String>> data) {
this.data = data;
}
public List<Map<String, String>> getOld() {
return old;
}
public void setOld(List<Map<String, String>> old) {
this.old = old;
}
/**
* 将Message转换为FlatMessage
*
* @param message 原生message
* @return FlatMessage列表
*/
public static List<FlatMessage> messageConverter(Message message) {
try {
if (message == null) {
return null;
}
List<FlatMessage> flatMessages = new ArrayList<>();
List<ByteString> rawEntries = message.getRawEntries();
for (ByteString byteString : rawEntries) {
CanalEntry.Entry entry = CanalEntry.Entry.parseFrom(byteString);
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
|| entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
continue;
}
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new RuntimeException(
"ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
e);
}
CanalEntry.EventType eventType = rowChange.getEventType();
FlatMessage flatMessage = new FlatMessage(message.getId());
flatMessages.add(flatMessage);
flatMessage.setDatabase(entry.getHeader().getSchemaName());
flatMessage.setTable(entry.getHeader().getTableName());
flatMessage.setIsDdl(rowChange.getIsDdl());
flatMessage.setType(eventType.toString());
flatMessage.setTs(System.currentTimeMillis());
flatMessage.setSql(rowChange.getSql());
if (!rowChange.getIsDdl()) {
Map<String, Integer> sqlType = new LinkedHashMap<>();
Map<String, String> mysqlType = new LinkedHashMap<>();
List<Map<String, String>> data = new ArrayList<>();
List<Map<String, String>> old = new ArrayList<>();
Set<String> updateSet = new HashSet<>();
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
&& eventType != CanalEntry.EventType.DELETE) {
continue;
}
Map<String, String> row = new LinkedHashMap<>();
List<CanalEntry.Column> columns;
if (eventType == CanalEntry.EventType.DELETE) {
columns = rowData.getBeforeColumnsList();
} else {
columns = rowData.getAfterColumnsList();
}
for (CanalEntry.Column column : columns) {
sqlType.put(column.getName(), column.getSqlType());
mysqlType.put(column.getName(), column.getMysqlType());
row.put(column.getName(), column.getValue());
// 获取update为true的字段
if (column.getUpdated()) {
updateSet.add(column.getName());
}
}
if (!row.isEmpty()) {
data.add(row);
}
if (eventType == CanalEntry.EventType.UPDATE) {
Map<String, String> rowOld = new LinkedHashMap<>();
for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
if (updateSet.contains(column.getName())) {
rowOld.put(column.getName(), column.getValue());
}
}
// update操作将记录修改前的值
if (!rowOld.isEmpty()) {
old.add(rowOld);
}
}
}
if (!sqlType.isEmpty()) {
flatMessage.setSqlType(sqlType);
}
if (!mysqlType.isEmpty()) {
flatMessage.setMysqlType(mysqlType);
}
if (!data.isEmpty()) {
flatMessage.setData(data);
}
if (!old.isEmpty()) {
flatMessage.setOld(old);
}
}
}
return flatMessages;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/**
* 将FlatMessage按指定的字段值hash拆分
*
* @param flatMessage flatMessage
* @param partitionsNum 分区数量
* @param pkHashConfig hash映射
* @return 拆分后的flatMessage数组
*/
public static FlatMessage[] messagePartition(FlatMessage flatMessage, Integer partitionsNum,
Map<String, String> pkHashConfig) {
if (partitionsNum == null) {
partitionsNum = 1;
}
FlatMessage[] partitionMessages = new FlatMessage[partitionsNum];
String pk = pkHashConfig.get(flatMessage.getDatabase() + "." + flatMessage.getTable());
if (pk == null || flatMessage.getIsDdl()) {
partitionMessages[0] = flatMessage;
} else {
if (flatMessage.getData() != null) {
int idx = 0;
for (Map<String, String> row : flatMessage.getData()) {
String value = row.get(pk);
if (value == null) {
value = "";
}
int hash = value.hashCode();
int pkHash = Math.abs(hash) % partitionsNum;
// math.abs可能返回负值，这里再取反，把出现负值的数据还是写到固定的分区，仍然可以保证消费顺序
pkHash = Math.abs(pkHash);
FlatMessage flatMessageTmp = partitionMessages[pkHash];
if (flatMessageTmp == null) {
flatMessageTmp = new FlatMessage(flatMessage.getId());
partitionMessages[pkHash] = flatMessageTmp;
flatMessageTmp.setDatabase(flatMessage.getDatabase());
flatMessageTmp.setTable(flatMessage.getTable());
flatMessageTmp.setIsDdl(flatMessage.getIsDdl());
flatMessageTmp.setType(flatMessage.getType());
flatMessageTmp.setSql(flatMessage.getSql());
flatMessageTmp.setSqlType(flatMessage.getSqlType());
flatMessageTmp.setMysqlType(flatMessage.getMysqlType());
}
List<Map<String, String>> data = flatMessageTmp.getData();
if (data == null) {
data = new ArrayList<>();
flatMessageTmp.setData(data);
}
data.add(row);
if (flatMessage.getOld() != null && !flatMessage.getOld().isEmpty()) {
List<Map<String, String>> old = flatMessageTmp.getOld();
if (old == null) {
old = new ArrayList<>();
flatMessageTmp.setOld(old);
}
old.add(flatMessage.getOld().get(idx));
}
idx++;
}
}
}
return partitionMessages;
}
@Override
public String toString() {
return "FlatMessage{" + "id=" + id + ", database='" + database + '\'' + ", table='" + table + '\'' + ", isDdl="
+ isDdl + ", type='" + type + '\'' + ", ts=" + ts + ", sql='" + sql + '\'' + ", sqlType=" + sqlType
+ ", mysqlType=" + mysqlType + ", data=" + data + ", old=" + old + '}';
}
}