From e8da1c884bce898732ec51868ea1b1a001dc121b Mon Sep 17 00:00:00 2001 From: kaimingwan <344277934@qq.com> Date: Fri, 22 Sep 2017 22:01:57 +0800 Subject: [PATCH 01/34] table meta support --- client/src/test/java/logback.xml | 2 +- .../tddl/dbsync/binlog/BinlogPosition.java | 146 +++++ .../resources/example/instance.properties | 43 +- deployer/src/main/resources/logback.xml | 2 +- .../src/main/resources/spring/dal-dao.xml | 42 ++ .../main/resources/spring/file-instance.xml | 43 ++ .../main/resources/spring/mybatis-config.xml | 20 + .../spring/sql-map/sqlmap_history.xml | 57 ++ .../spring/sql-map/sqlmap_snapshot.xml | 66 ++ .../main/resources/spring/sqlmap-config.xml | 9 + .../parse/driver/mysql/MysqlConnector.java | 9 + parse/pom.xml | 19 + .../parse/inbound/AbstractEventParser.java | 6 +- .../canal/parse/inbound/BinlogParser.java | 2 +- .../canal/parse/inbound/ErosaConnection.java | 2 - .../otter/canal/parse/inbound/TableMeta.java | 149 +++-- .../mysql/AbstractMysqlEventParser.java | 40 ++ .../inbound/mysql/LocalBinlogEventParser.java | 2 +- .../parse/inbound/mysql/MysqlConnection.java | 99 +++ .../parse/inbound/mysql/MysqlEventParser.java | 14 +- .../inbound/mysql/dbsync/DBMSAction.java | 77 +++ .../inbound/mysql/dbsync/LogEventConvert.java | 70 ++- .../inbound/mysql/dbsync/TableMetaCache.java | 234 +++++--- .../parse/inbound/mysql/ddl/DdlResult.java | 116 ++++ .../inbound/mysql/ddl/DruidDdlParser.java | 201 +++++++ .../mysql/tsdb/DataSourceFactoryTSDB.java | 43 ++ .../inbound/mysql/tsdb/MemoryTableMeta.java | 223 +++++++ .../inbound/mysql/tsdb/TableMetaManager.java | 565 ++++++++++++++++++ .../inbound/mysql/tsdb/TableMetaTSDB.java | 37 ++ .../mysql/tsdb/dao/MetaHistoryDAO.java | 44 ++ .../mysql/tsdb/dao/MetaSnapshotDAO.java | 48 ++ .../mysql/tsdb/model/MetaHistoryDO.java | 141 +++++ .../mysql/tsdb/model/MetaSnapshotDO.java | 105 ++++ parse/src/main/resources/dal-dao.xml | 42 ++ parse/src/main/resources/mybatis-config.xml | 20 + .../main/resources/sql-map/sqlmap_history.xml | 57 ++ .../resources/sql-map/sqlmap_snapshot.xml | 66 ++ parse/src/main/resources/sqlmap-config.xml | 9 + .../parse/inbound/TableMetaCacheTest.java | 38 +- .../inbound/group/GroupEventPaserTest.java | 5 +- .../inbound/mysql/LocalBinlogDumpTest.java | 6 +- .../mysql/LocalBinlogEventParserTest.java | 16 +- .../parse/inbound/mysql/MysqlDumpTest.java | 4 +- .../inbound/mysql/MysqlEventParserTest.java | 6 +- .../mysql/tsdb/MemoryTableMetaTest.java | 38 ++ .../mysql/tsdb/MetaHistoryDAOTest.java | 36 ++ .../mysql/tsdb/TableMetaManagerTest.java | 47 ++ .../resources/binlog/tsdb/mysql-bin.000001 | Bin 0 -> 178 bytes .../resources/binlog/tsdb/mysql-bin.000002 | Bin 0 -> 4809 bytes .../resources/binlog/tsdb/mysql-bin.000003 | Bin 0 -> 14377 bytes parse/src/test/resources/dal-dao.xml | 46 ++ parse/src/test/resources/ddl/create.sql | 23 + pom.xml | 18 +- .../BaseCanalServerWithEmbededTest.java | 8 +- ...analServerWithEmbedded_StandaloneTest.java | 4 +- 55 files changed, 2931 insertions(+), 234 deletions(-) create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java create mode 100644 deployer/src/main/resources/spring/dal-dao.xml create mode 100644 deployer/src/main/resources/spring/mybatis-config.xml create mode 100644 deployer/src/main/resources/spring/sql-map/sqlmap_history.xml create mode 100644 deployer/src/main/resources/spring/sql-map/sqlmap_snapshot.xml create mode 100644 deployer/src/main/resources/spring/sqlmap-config.xml create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java create mode 100644 parse/src/main/resources/dal-dao.xml create mode 100644 parse/src/main/resources/mybatis-config.xml create mode 100644 parse/src/main/resources/sql-map/sqlmap_history.xml create mode 100644 parse/src/main/resources/sql-map/sqlmap_snapshot.xml create mode 100644 parse/src/main/resources/sqlmap-config.xml create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java create mode 100644 parse/src/test/resources/binlog/tsdb/mysql-bin.000001 create mode 100644 parse/src/test/resources/binlog/tsdb/mysql-bin.000002 create mode 100644 parse/src/test/resources/binlog/tsdb/mysql-bin.000003 create mode 100644 parse/src/test/resources/dal-dao.xml create mode 100644 parse/src/test/resources/ddl/create.sql diff --git a/client/src/test/java/logback.xml b/client/src/test/java/logback.xml index df46c256..352865df 100644 --- a/client/src/test/java/logback.xml +++ b/client/src/test/java/logback.xml @@ -8,7 +8,7 @@ - + \ No newline at end of file diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java new file mode 100644 index 00000000..92bba817 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java @@ -0,0 +1,146 @@ +package com.taobao.tddl.dbsync.binlog; + +import com.taobao.tddl.dbsync.binlog.LogPosition; + +/** + * Position inside binlog file + * + * @author Seppo Jaakola + * @version 1.0 + */ +public class BinlogPosition extends LogPosition { + + /* The source server_id of position, 0 invalid */ + protected final long masterId; + + /* The timestamp, in seconds, 0 invalid */ + protected final long timestamp; + + public BinlogPosition(String fileName, long position, long masterId, long timestamp){ + super(fileName, position); + this.masterId = masterId; + this.timestamp = timestamp; + } + + public BinlogPosition(LogPosition logPosition, long masterId, long timestamp){ + super(logPosition.getFileName(), logPosition.getPosition()); + this.masterId = masterId; + this.timestamp = timestamp; + } + + public BinlogPosition(BinlogPosition binlogPosition){ + super(binlogPosition.getFileName(), binlogPosition.getPosition()); + this.masterId = binlogPosition.masterId; + this.timestamp = binlogPosition.timestamp; + } + + private final static long[] pow10 = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, + 10000000000L, 100000000000L, 1000000000000L, 10000000000000L, + 100000000000000L, 1000000000000000L, 10000000000000000L, 100000000000000000L, + 1000000000000000000L }; + + public static String placeHolder(int bit, long number) { + if (bit > 18) { + throw new IllegalArgumentException("Bit must less than 18, but given " + bit); + } + + final long max = pow10[bit]; + if (number >= max) { + // 当 width < 数值的最大位数时，应该直接返回数值 + return String.valueOf(number); + } + + return String.valueOf(max + number).substring(1); + } + + /** + * Return BinlogPosition in String representation. This serves as EventId for DBMSEvent. + */ + public String format2String(final int positionMaxLen) { + String binlogSuffix = fileName; + String binlogOffset = placeHolder((int) positionMaxLen, position); + // 输出 '000001:0000000004@12+12314130' + StringBuffer buf = new StringBuffer(40); + buf.append(binlogSuffix); + buf.append(':'); + buf.append(binlogOffset); + if (masterId != 0) { + buf.append('#'); + buf.append(masterId); + } + if (timestamp != 0) { + buf.append('.'); + buf.append(timestamp); + } + return buf.toString(); + } + + public static BinlogPosition parseFromString(String source) { + int colonIndex = source.indexOf(':'); + int miscIndex = colonIndex + 1; + int sharpIndex = source.indexOf('#', miscIndex); + int semicolonIndex = source.indexOf(';', miscIndex); // NOTE: 向后兼容 + int dotIndex = source.lastIndexOf('.'); + if (colonIndex == -1) { + return null; // NOTE: 错误的位点 + } + + String binlogSuffix = source.substring(0, colonIndex); + long binlogPosition; + if (sharpIndex != -1) { + binlogPosition = Long.parseLong(source.substring(miscIndex, sharpIndex)); + } else if (semicolonIndex != -1) { + binlogPosition = Long.parseLong(source.substring(miscIndex, semicolonIndex)); // NOTE: 向后兼容 + } else if (dotIndex != -1) { + binlogPosition = Long.parseLong(source.substring(miscIndex, dotIndex)); + } else { + binlogPosition = Long.parseLong(source.substring(miscIndex)); + } + + long masterId = 0; // NOTE: 默认值为 0 + if (sharpIndex != -1) { + if (dotIndex != -1) { + masterId = Long.parseLong(source.substring(sharpIndex + 1, dotIndex)); + } else { + masterId = Long.parseLong(source.substring(sharpIndex + 1)); + } + } + + long timestamp = 0; // NOTE: 默认值为 0 + if (dotIndex != -1 && dotIndex > colonIndex) { + timestamp = Long.parseLong(source.substring(dotIndex + 1)); + } + + return new BinlogPosition(binlogSuffix, binlogPosition, // NL + masterId, + timestamp); + } + + public String getFilePattern() { + final int index = fileName.indexOf('.'); + if (index != -1) { + return fileName.substring(0, index); + } + return null; + } + + public void setFilePattern(String filePattern) { + // We tolerate the event ID with or without the binlog prefix. + if (fileName.indexOf('.') < 0) { + fileName = filePattern + '.' + fileName; + } + } + + public long getMasterId() { + return masterId; + } + + public long getTimestamp() { + return timestamp; + } + + @Override + public String toString() { + return format2String(10); + } +} diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 2bdfffb2..3c5e0a8e 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -1,27 +1,36 @@ ################################################# ## mysql serverId -canal.instance.mysql.slaveId = 1234 - +canal.instance.mysql.slaveId=1234 # position info -canal.instance.master.address = 127.0.0.1:3306 -canal.instance.master.journal.name = -canal.instance.master.position = -canal.instance.master.timestamp = +canal.instance.master.address=127.0.0.1:3306 +canal.instance.master.journal.name=mysql-bin.000004 +canal.instance.master.position=25678 +canal.instance.master.timestamp=1506088042 -#canal.instance.standby.address = + +# tsdb info +canal.instance.tsdb.address=127.0.0.1:3306 +canal.instance.tsdb.enable=true +canal.instance.tsdb.dbUsername=canal +canal.instance.tsdb.dbPassword=canal +# tsdb关联的目标db，和canal.instance.defaultDatabaseName意义相同 +canal.instance.tsdb.defaultDatabaseName=test +canal.instance.tsdb.connectionCharset=UTF-8 +canal.instance.tsdb.driverClassName=com.mysql.jdbc.Driver +canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/tsdb + + +#canal.instance.standby.address = #canal.instance.standby.journal.name = #canal.instance.standby.position = #canal.instance.standby.timestamp = - # username/password -canal.instance.dbUsername = canal -canal.instance.dbPassword = canal -canal.instance.defaultDatabaseName = -canal.instance.connectionCharset = UTF-8 - +canal.instance.dbUsername=canal +canal.instance.dbPassword=canal +canal.instance.defaultDatabaseName=test +canal.instance.connectionCharset=UTF-8 # table regex -canal.instance.filter.regex = .*\\..* -# table black regex -canal.instance.filter.black.regex = - +canal.instance.filter.regex=.*\\..* +# table black regex , 默认过滤表结构管理的数据 +canal.instance.filter.black.regex=tsdb\\..* ################################################# \ No newline at end of file diff --git a/deployer/src/main/resources/logback.xml b/deployer/src/main/resources/logback.xml index 3d1e7a69..1b0f4103 100644 --- a/deployer/src/main/resources/logback.xml +++ b/deployer/src/main/resources/logback.xml @@ -75,7 +75,7 @@ - + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/dal-dao.xml b/deployer/src/main/resources/spring/dal-dao.xml new file mode 100644 index 00000000..644979fc --- /dev/null +++ b/deployer/src/main/resources/spring/dal-dao.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index b5fcc908..a2563119 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -167,5 +167,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deployer/src/main/resources/spring/mybatis-config.xml b/deployer/src/main/resources/spring/mybatis-config.xml new file mode 100644 index 00000000..83ac0116 --- /dev/null +++ b/deployer/src/main/resources/spring/mybatis-config.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/sql-map/sqlmap_history.xml b/deployer/src/main/resources/spring/sql-map/sqlmap_history.xml new file mode 100644 index 00000000..00197194 --- /dev/null +++ b/deployer/src/main/resources/spring/sql-map/sqlmap_history.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + insert into `canal_table_meta_history` () + values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#schema#,#table#,#sql#,#type#,#extra#); + + SELECT last_insert_id() + + + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/sql-map/sqlmap_snapshot.xml b/deployer/src/main/resources/spring/sql-map/sqlmap_snapshot.xml new file mode 100644 index 00000000..9065a7e0 --- /dev/null +++ b/deployer/src/main/resources/spring/sql-map/sqlmap_snapshot.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + insert into `canal_table_meta_snapshot` () + values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#); + + SELECT last_insert_id() + + + + + update `canal_table_meta_snapshot` set gmt_modified=now(), + binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra# + where binlog_timestamp=0 + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/sqlmap-config.xml b/deployer/src/main/resources/spring/sqlmap-config.xml new file mode 100644 index 00000000..983e9663 --- /dev/null +++ b/deployer/src/main/resources/spring/sqlmap-config.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index a05c3b19..ba4c51a7 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -35,6 +35,7 @@ public class MysqlConnector { private byte charsetNumber = 33; private String defaultSchema = "retl"; private int soTimeout = 30 * 1000; + private int connTimeout = 5 * 1000; private int receiveBufferSize = 16 * 1024; private int sendBufferSize = 16 * 1024; @@ -129,6 +130,7 @@ public class MysqlConnector { connector.setReceiveBufferSize(getReceiveBufferSize()); connector.setSendBufferSize(getSendBufferSize()); connector.setSoTimeout(getSoTimeout()); + connector.setConnTimeout(connTimeout); return connector; } @@ -325,4 +327,11 @@ public class MysqlConnector { this.dumping = dumping; } + public int getConnTimeout() { + return connTimeout; + } + + public void setConnTimeout(int connTimeout) { + this.connTimeout = connTimeout; + } } diff --git a/parse/pom.xml b/parse/pom.xml index d80342f9..0adf86e0 100644 --- a/parse/pom.xml +++ b/parse/pom.xml @@ -45,11 +45,30 @@ canal.parse.driver ${project.version} + + com.alibaba + druid + + + mysql + mysql-connector-java + + + org.apache.ibatis + ibatis-sqlmap + 2.3.4.726 + junit junit test + + org.springframework + spring-test + test + + diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index 6b34edf5..8f6ea20d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -173,7 +173,7 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle public boolean sink(EVENT event) { try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,false); if (!running) { return false; @@ -320,13 +320,13 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle return result; } - protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception { + protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod,boolean isSeek) throws Exception { long startTs = -1; boolean enabled = getProfilingEnabled(); if (enabled) { startTs = System.currentTimeMillis(); } - CanalEntry.Entry event = binlogParser.parse(bod); + CanalEntry.Entry event = binlogParser.parse(bod,isSeek); if (enabled) { this.parsingInterval = System.currentTimeMillis() - startTs; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java index 3ff91287..01c55a51 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java @@ -11,7 +11,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry; */ public interface BinlogParser extends CanalLifeCycle { - CanalEntry.Entry parse(T event) throws CanalParseException; + CanalEntry.Entry parse(T event,boolean isSeek) throws CanalParseException; void reset(); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java index 4b32bb94..2e953956 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java @@ -15,8 +15,6 @@ public interface ErosaConnection { public void disconnect() throws IOException; - public boolean isConnected(); - /** * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据 */ diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java index de934fbc..d3009624 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java @@ -3,51 +3,79 @@ package com.alibaba.otter.canal.parse.inbound; import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang.StringUtils; - import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent; +import org.apache.commons.lang.StringUtils; /** * 描述数据meta对象,mysql binlog中对应的{@linkplain TableMapLogEvent}包含的信息不全 - * + * * 
  * 1. 主键信息
  * 2. column name
  * 3. unsigned字段
  *
- * + * * @author jianghang 2013-1-18 下午12:24:59 * @version 1.0.0 */ public class TableMeta { - private String fullName; // schema.table - private List fileds; + private String schema; + private String table; + private List fields = new ArrayList(); + private String ddl; // 表结构的DDL语句 - public TableMeta(String fullName, List fileds){ - this.fullName = fullName; - this.fileds = fileds; + public TableMeta() { + + } + + public TableMeta(String schema, String table, List fields) { + this.schema = schema; + this.table = table; + this.fields = fields; } public String getFullName() { - return fullName; + return schema + "." + table; } - public void setFullName(String fullName) { - this.fullName = fullName; + public String getSchema() { + return schema; } - public List getFileds() { - return fileds; + public void setSchema(String schema) { + this.schema = schema; } - public void setFileds(List fileds) { - this.fileds = fileds; + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public List getFields() { + return fields; + } + + public void setFields(List fileds) { + this.fields = fileds; + } + + public FieldMeta getFieldMetaByName(String name) { + for (FieldMeta meta : fields) { + if (meta.getColumnName().equalsIgnoreCase(name)) { + return meta; + } + } + + throw new RuntimeException("unknow column : " + name); } public List getPrimaryFields() { List primarys = new ArrayList(); - for (FieldMeta meta : fileds) { + for (FieldMeta meta : fields) { if (meta.isKey()) { primarys.add(meta); } @@ -56,12 +84,47 @@ public class TableMeta { return primarys; } + public String getDdl() { + return ddl; + } + + public void setDdl(String ddl) { + this.ddl = ddl; + } + + public void addFieldMeta(FieldMeta fieldMeta) { + this.fields.add(fieldMeta); + } + + @Override + public String toString() { + StringBuilder data = new StringBuilder(); + data.append("TableMeta [schema=" + schema + ", table=" + table + ", fileds="); + for (FieldMeta field : fields) { + data.append("\n\t").append(field.toString()); + } + data.append("\n]"); + return data.toString(); + } + public static class FieldMeta { + public FieldMeta() { + + } + + public FieldMeta(String columnName, String columnType, boolean nullable, boolean key, String defaultValue) { + this.columnName = columnName; + this.columnType = columnType; + this.nullable = nullable; + this.key = key; + this.defaultValue = defaultValue; + } + private String columnName; private String columnType; - private String isNullable; - private String iskey; + private boolean nullable; + private boolean key; private String defaultValue; private String extra; @@ -81,20 +144,8 @@ public class TableMeta { this.columnType = columnType; } - public String getIsNullable() { - return isNullable; - } - - public void setIsNullable(String isNullable) { - this.isNullable = isNullable; - } - - public String getIskey() { - return iskey; - } - - public void setIskey(String iskey) { - this.iskey = iskey; + public void setNullable(boolean nullable) { + this.nullable = nullable; } public String getDefaultValue() { @@ -105,6 +156,22 @@ public class TableMeta { this.defaultValue = defaultValue; } + public boolean isUnsigned() { + return StringUtils.containsIgnoreCase(columnType, "unsigned"); + } + + public boolean isNullable() { + return nullable; + } + + public boolean isKey() { + return key; + } + + public void setKey(boolean key) { + this.key = key; + } + public String getExtra() { return extra; } @@ -113,22 +180,10 @@ public class TableMeta { this.extra = extra; } - public boolean isUnsigned() { - return StringUtils.containsIgnoreCase(columnType, "unsigned"); - } - - public boolean isKey() { - return StringUtils.equalsIgnoreCase(iskey, "PRI"); - } - - public boolean isNullable() { - return StringUtils.equalsIgnoreCase(isNullable, "YES"); - } - public String toString() { return "FieldMeta [columnName=" + columnName + ", columnType=" + columnType + ", defaultValue=" - + defaultValue + ", extra=" + extra + ", isNullable=" + isNullable + ", iskey=" + iskey + "]"; + + defaultValue + ", nullable=" + nullable + ", key=" + key + "]"; } - } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index c8bf197b..4510159b 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -2,6 +2,9 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.nio.charset.Charset; +import javax.annotation.Resource; + +import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,6 +13,7 @@ import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.BinlogParser; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; public abstract class AbstractMysqlEventParser extends AbstractEventParser { @@ -25,6 +29,10 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { protected boolean filterRows = false; protected boolean filterTableError = false; + @Resource + protected TableMetaManager tableMetaManager; + protected boolean useDruidDdlFilter = true; + protected BinlogParser buildParser() { LogEventConvert convert = new LogEventConvert(); if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) { @@ -41,6 +49,9 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { convert.setFilterQueryDdl(filterQueryDdl); convert.setFilterRows(filterRows); convert.setFilterTableError(filterTableError); + + //初始化parser的时候也初始化管理mysql 表结构的管理器 + tableMetaManager.init(); return convert; } @@ -53,6 +64,20 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { } } + /** + * 回滚到指定位点 + * @param position + * @return + */ + protected boolean processTableMeta(BinlogPosition position) { + if (tableMetaManager != null) { + return tableMetaManager.rollback(position); + } + + return true; + } + + public void setEventBlackFilter(CanalEventFilter eventBlackFilter) { super.setEventBlackFilter(eventBlackFilter); @@ -97,4 +122,19 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { this.filterTableError = filterTableError; } + public TableMetaManager getTableMetaManager() { + return tableMetaManager; + } + + public void setTableMetaManager(TableMetaManager tableMetaManager) { + this.tableMetaManager = tableMetaManager; + } + + public boolean isUseDruidDdlFilter() { + return useDruidDdlFilter; + } + + public void setUseDruidDdlFilter(boolean useDruidDdlFilter) { + this.useDruidDdlFilter = useDruidDdlFilter; + } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java index 331b4d45..87f31208 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java @@ -50,7 +50,7 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements throw new CanalParseException(e); } - tableMetaCache = new TableMetaCache(metaConnection); + tableMetaCache = new TableMetaCache(metaConnection,tableMetaManager); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index c648c10d..98263792 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -3,8 +3,14 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.Charset; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; +import java.util.Properties; +import com.mysql.jdbc.Driver; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; @@ -21,6 +27,8 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.ErosaConnection; import com.alibaba.otter.canal.parse.inbound.SinkFunction; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; +import com.alibaba.otter.canal.parse.support.AuthenticationInfo; + import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; @@ -35,20 +43,59 @@ public class MysqlConnection implements ErosaConnection { private BinlogFormat binlogFormat; private BinlogImage binlogImage; + // tsdb releated + private AuthenticationInfo authInfo; + private Connection conn; + protected int connTimeout = 5 * 1000; // 5秒 + protected int soTimeout = 60 * 60 * 1000; // 1小时 + protected int bufferSize = 16 * 1024; + + public MysqlConnection(){ } public MysqlConnection(InetSocketAddress address, String username, String password){ + authInfo = new AuthenticationInfo(); + authInfo.setAddress(address); + authInfo.setUsername(username); + authInfo.setPassword(password); connector = new MysqlConnector(address, username, password); + //将connection里面的参数透传下 + connector.setSoTimeout(soTimeout); + connector.setConnTimeout(connTimeout); } public MysqlConnection(InetSocketAddress address, String username, String password, byte charsetNumber, String defaultSchema){ + authInfo = new AuthenticationInfo(); + authInfo.setAddress(address); + authInfo.setUsername(username); + authInfo.setPassword(password); + authInfo.setDefaultDatabaseName(defaultSchema); connector = new MysqlConnector(address, username, password, charsetNumber, defaultSchema); + //将connection里面的参数透传下 + connector.setSoTimeout(soTimeout); + connector.setConnTimeout(connTimeout); } public void connect() throws IOException { connector.connect(); + //准备一个connection连接给tsdb查询用 + Properties info = new Properties(); + info.put("user", authInfo.getUsername()); + info.put("password", authInfo.getPassword()); + info.put("connectTimeout", String.valueOf(connTimeout)); + info.put("socketTimeout", String.valueOf(soTimeout)); + String url = "jdbc:mysql://" + authInfo.getAddress().getHostName() + ":" + + String.valueOf(authInfo.getAddress().getPort()) + "?allowMultiQueries=true"; + try { + Driver driver = new com.mysql.jdbc.Driver(); + conn = driver.connect(url, info); + // conn = DriverManager.getConnection(url, info); + } catch (SQLException e) { + throw new CanalParseException(e); + } + } public void reconnect() throws IOException { @@ -68,6 +115,25 @@ public class MysqlConnection implements ErosaConnection { return exector.query(cmd); } + public T query(String sql, ProcessJdbcResult processor) { + Statement stmt = null; + try { + stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql); + return processor.process(rs); + } catch (SQLException e) { + throw new CanalParseException(e); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // ignore + } + } + } + } + public void update(String cmd) throws IOException { MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector); exector.update(cmd); @@ -147,6 +213,8 @@ public class MysqlConnection implements ErosaConnection { connection.setCharset(getCharset()); connection.setSlaveId(getSlaveId()); connection.setConnector(connector.fork()); + //set authInfo + connection.setAuthInfo(authInfo); return connection; } @@ -382,4 +450,35 @@ public class MysqlConnection implements ErosaConnection { return binlogImage; } + public Connection getConn() { + return conn; + } + public InetSocketAddress getAddress() { + return authInfo.getAddress(); + } + + public void setConnTimeout(int connTimeout) { + this.connTimeout = connTimeout; + } + + public void setSoTimeout(int soTimeout) { + this.soTimeout = soTimeout; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } + + public AuthenticationInfo getAuthInfo() { + return authInfo; + } + + public void setAuthInfo(AuthenticationInfo authInfo) { + this.authInfo = authInfo; + } + + public static interface ProcessJdbcResult { + + public T process(ResultSet rs) throws SQLException; + } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 9b7b152a..1e608faa 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -69,6 +69,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // update by yishun.chen,特殊异常处理参数 private int dumpErrorCount = 0; // binlogDump失败异常计数 + private int dumpTimeoutCount = 0;// socketTimeout异常 private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 protected ErosaConnection buildErosaConnection() { @@ -118,7 +119,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } } - tableMetaCache = new TableMetaCache(metaConnection); + if(tableMetaManager != null){ + tableMetaManager.setConnection(metaConnection); + tableMetaManager.setFilter(eventFilter); + } + + tableMetaCache = new TableMetaCache(metaConnection,tableMetaManager); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } } @@ -438,7 +444,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE public boolean sink(LogEvent event) { try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,true); if (entry == null) { return true; } @@ -471,7 +477,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE public boolean sink(LogEvent event) { try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,true); if (entry == null) { return true; } @@ -683,7 +689,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE public boolean sink(LogEvent event) { EntryPosition entryPosition = null; try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,true); if (entry == null) { return true; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java new file mode 100644 index 00000000..4efbdc64 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java @@ -0,0 +1,77 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.dbsync; + +/** + * Defines database change action types: INSERT, UPDATE, DELETE, OTHER. + * + * @author Changyuan.lh + * @version 1.0 + */ +public enum DBMSAction { + + INSERT('I'), UPDATE('U'), DELETE('D'), REPLACE('R'), OTHER('O'), CREATE('C'), ALTER('A'), ERASE('E'), QUERY('Q'), + ROWQUERY('W'), TRUNCATE('T'), CINDEX('X'), DINDEX('Y'), RENAME('Z'); + + protected final byte bValue; + + DBMSAction(char ch){ + this.bValue = (byte) ch; + } + + /** + * Return action type from byte value. + */ + public static DBMSAction fromValue(int iValue) { + switch ((char) iValue) { + case 'I': + case 'M': // MERGE (Oracle only) + return INSERT; + case 'U': + return UPDATE; + case 'D': // DELETE + return DELETE; + case 'R': // REPLACE + return REPLACE; + case 'C': + return CREATE; + case 'A': + return ALTER; + case 'E': + return ERASE; + case 'Q': + return QUERY; + case 'W': + return ROWQUERY; + case 'T': + return TRUNCATE; + case 'X': + return CINDEX; + case 'Y': + return DINDEX; + case 'Z': + return RENAME; + } + return OTHER; + } + + /** + * Return action type from query. + */ + public static DBMSAction fromQuery(String query) { + int length = query.length(); + for (int index = 0; index < length; index++) { + char ch = query.charAt(index); + if (!Character.isWhitespace(ch)) { + return DBMSAction.fromValue(ch); + } + } + + return OTHER; + } + + /** + * Return byte value of action type. + */ + public byte value() { + return bValue; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 1bbb0eb8..222d3010 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -9,6 +9,7 @@ import java.sql.Types; import java.util.BitSet; import java.util.List; +import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; @@ -87,7 +88,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar // 新增rows过滤，用于仅订阅除rows以外的数据 private boolean filterRows = false; - public Entry parse(LogEvent logEvent) throws CanalParseException { + @Override + public Entry parse(LogEvent logEvent, boolean isSeek) throws CanalParseException { if (logEvent == null || logEvent instanceof UnknownLogEvent) { return null; } @@ -98,7 +100,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar binlogFileName = ((RotateLogEvent) logEvent).getFilename(); break; case LogEvent.QUERY_EVENT: - return parseQueryEvent((QueryLogEvent) logEvent); + return parseQueryEvent((QueryLogEvent) logEvent,isSeek); case LogEvent.XID_EVENT: return parseXidEvent((XidLogEvent) logEvent); case LogEvent.TABLE_MAP_EVENT: @@ -137,7 +139,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } - private Entry parseQueryEvent(QueryLogEvent event) { + private Entry parseQueryEvent(QueryLogEvent event, boolean isSeek) { String queryString = event.getQuery(); if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) { TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId()); @@ -215,24 +217,28 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } // 更新下table meta cache - if (tableMetaCache != null - && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) { - for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) { - String schemaName0 = event.getDbName(); // 防止rename语句后产生schema变更带来影响 - if (StringUtils.isNotEmpty(renameResult.getSchemaName())) { - schemaName0 = renameResult.getSchemaName(); - } + //if (tableMetaCache != null + // && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) { + // for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) { + // String schemaName0 = event.getDbName(); // 防止rename语句后产生schema变更带来影响 + // if (StringUtils.isNotEmpty(renameResult.getSchemaName())) { + // schemaName0 = renameResult.getSchemaName(); + // } + // + // tableName = renameResult.getTableName(); + // if (StringUtils.isNotEmpty(tableName)) { + // // 如果解析到了正确的表信息，则根据全名进行清除 + // tableMetaCache.clearTableMeta(schemaName0, tableName); + // } else { + // // 如果无法解析正确的表信息，则根据schema进行清除 + // tableMetaCache.clearTableMetaWithSchemaName(schemaName0); + // } + // } + //} - tableName = renameResult.getTableName(); - if (StringUtils.isNotEmpty(tableName)) { - // 如果解析到了正确的表信息，则根据全名进行清除 - tableMetaCache.clearTableMeta(schemaName0, tableName); - } else { - // 如果无法解析正确的表信息，则根据schema进行清除 - tableMetaCache.clearTableMetaWithSchemaName(schemaName0); - } - } - } + //使用新的表结构元数据管理方式 + BinlogPosition position = createPosition(event.getHeader()); + tableMetaCache.apply(position, event.getDbName(), queryString); Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); @@ -348,6 +354,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar table.getDbName(), table.getTableName(), eventType); + BinlogPosition position = createPosition(event.getHeader()); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); rowChangeBuider.setTableId(event.getTableId()); rowChangeBuider.setIsDdl(false); @@ -359,7 +366,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar boolean tableError = false; TableMeta tableMeta = null; if (tableMetaCache != null) {// 入错存在table meta cache - tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true); + tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true,position); if (tableMeta == null) { tableError = true; if (!filterTableError) { @@ -405,6 +412,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } + private BinlogPosition createPosition(LogHeader logHeader) { + return new BinlogPosition(binlogFileName, logHeader.getLogPos(), logHeader.getServerId(), logHeader.getWhen()); // 记录到秒 + } + private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException { int columnCnt = event.getTable().getColumnCnt(); @@ -413,18 +424,19 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar boolean tableError = false; // check table fileds count，只能处理加字段 boolean existRDSNoPrimaryKey = false; - if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) { + if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) { if (tableMetaCache.isOnRDS()) { // 特殊处理下RDS的场景 List primaryKeys = tableMeta.getPrimaryFields(); if (primaryKeys == null || primaryKeys.isEmpty()) { - if (columnInfo.length == tableMeta.getFileds().size() + 1 + if (columnInfo.length == tableMeta.getFields().size() + 1 && columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) { existRDSNoPrimaryKey = true; } } } + BinlogPosition position = createPosition(event.getHeader()); if (!existRDSNoPrimaryKey) { // online ddl增加字段操作步骤： // 1. 新增一张临时表，将需要做ddl表的数据全量导入 @@ -432,7 +444,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作 // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作 // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作 - tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false);// 强制重新获取一次 + tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false,position);// 强制重新获取一次 if (tableMeta == null) { tableError = true; if (!filterTableError) { @@ -442,11 +454,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } // 在做一次判断 - if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) { + if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) { tableError = true; if (!filterTableError) { throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() - + "," + columnInfo.length + " vs " + tableMeta.getFileds().size()); + + "," + columnInfo.length + " vs " + tableMeta.getFields().size()); } } } else { @@ -473,7 +485,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar FieldMeta fieldMeta = null; if (tableMeta != null && !tableError) { // 处理file meta - fieldMeta = tableMeta.getFileds().get(i); + fieldMeta = tableMeta.getFields().get(i); columnBuilder.setName(fieldMeta.getColumnName()); columnBuilder.setIsKey(fieldMeta.isKey()); // 增加mysql type类型,issue 73 @@ -672,9 +684,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return true; } - private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) { + private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, BinlogPosition position) { try { - return tableMetaCache.getTableMeta(dbName, tbName, useCache); + return tableMetaCache.getTableMeta(dbName, tbName, useCache,position); } catch (Exception e) { String message = ExceptionUtils.getRootCauseMessage(e); if (filterTableError) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index d8575ec8..ff0e720f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -1,138 +1,202 @@ package com.alibaba.otter.canal.parse.inbound.mysql.dbsync; import java.io.IOException; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.lang.StringUtils; -import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; -import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.google.common.base.Function; -import com.google.common.collect.MigrateMap; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.ProcessJdbcResult; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; /** * 处理table meta解析和缓存 - * + * * @author jianghang 2013-1-17 下午10:15:16 * @version 1.0.0 */ public class TableMetaCache { - public static final String COLUMN_NAME = "COLUMN_NAME"; - public static final String COLUMN_TYPE = "COLUMN_TYPE"; - public static final String IS_NULLABLE = "IS_NULLABLE"; - public static final String COLUMN_KEY = "COLUMN_KEY"; - public static final String COLUMN_DEFAULT = "COLUMN_DEFAULT"; - public static final String EXTRA = "EXTRA"; - private MysqlConnection connection; - private boolean isOnRDS = false; + public static final String COLUMN_NAME = "COLUMN_NAME"; + public static final String COLUMN_TYPE = "COLUMN_TYPE"; + public static final String IS_NULLABLE = "IS_NULLABLE"; + public static final String COLUMN_KEY = "COLUMN_KEY"; + public static final String COLUMN_DEFAULT = "COLUMN_DEFAULT"; + public static final String EXTRA = "EXTRA"; + private MysqlConnection connection; + private boolean isOnRDS = false; + + private TableMetaManager tableMetaManager; + // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表 + private LoadingCache tableMetaDB; + // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表 private Map tableMetaCache; - public TableMetaCache(MysqlConnection con){ - this.connection = con; - tableMetaCache = MigrateMap.makeComputingMap(new Function() { + /** + * 从db获取表结构 + * @param fullname + * @return + */ + private TableMeta getTableMetaByDB(final String fullname) { + return connection.query("desc " + fullname, new ProcessJdbcResult() { - public TableMeta apply(String name) { - try { - return getTableMeta0(name); - } catch (IOException e) { - // 尝试做一次retry操作 + @Override + public TableMeta process(ResultSet rs) throws SQLException { + List metas = new ArrayList(); + while (rs.next()) { + FieldMeta meta = new FieldMeta(); + // 做一个优化，使用String.intern()，共享String对象，减少内存使用 + meta.setColumnName(rs.getString("Field")); + meta.setColumnType(rs.getString("Type")); + meta.setNullable(StringUtils.equalsIgnoreCase(rs.getString("Null"), "YES")); + meta.setKey("PRI".equalsIgnoreCase(rs.getString("Key"))); + meta.setDefaultValue(rs.getString("Default")); + metas.add(meta); + } + + String[] names = StringUtils.split(fullname, "`.`"); + String schema = names[0]; + String table = names[1].substring(0, names[1].length()); + return new TableMeta(schema, table, metas); + } + }); + } + + public TableMetaCache(MysqlConnection con,TableMetaManager tableMetaManager) { + this.connection = con; + this.tableMetaManager = tableMetaManager; + //如果持久存储的表结构为空，从db里面获取下 + if (tableMetaManager == null) { + this.tableMetaDB = CacheBuilder.newBuilder().build(new CacheLoader() { + + @Override + public TableMeta load(String name) throws Exception { try { - connection.reconnect(); - return getTableMeta0(name); - } catch (IOException e1) { - throw new CanalParseException("fetch failed by table meta:" + name, e1); + return getTableMetaByDB(name); + } catch (CanalParseException e) { + // 尝试做一次retry操作 + try { + connection.reconnect(); + return getTableMetaByDB(name); + } catch (IOException e1) { + throw new CanalParseException("fetch failed by table meta:" + name, e1); + } } } - } + }); + } + + isOnRDS = connection.query("show global variables like 'rds\\_%'", new ProcessJdbcResult() { + + @Override + public Boolean process(ResultSet rs) throws SQLException { + if (rs.next()) { + return true; + } + return false; + } }); + } - try { - ResultSetPacket packet = connection.query("show global variables like 'rds\\_%'"); - if (packet.getFieldValues().size() > 0) { - isOnRDS = true; + public TableMeta getTableMeta(String schema, String table, BinlogPosition position) { + return getTableMeta(schema, table, true, position); + } + + public TableMeta getTableMeta(String schema, String table, boolean useCache, BinlogPosition position) { + TableMeta tableMeta = null; + if (tableMetaManager != null) { + tableMeta = tableMetaManager.find(schema, table); + if (tableMeta == null) { + // 因为条件变化，可能第一次的tableMeta没取到，需要从db获取一次，并记录到snapshot中 + String createDDL = connection.query("show create table " + getFullName(schema, table), + new ProcessJdbcResult() { + + @Override + public String process(ResultSet rs) throws SQLException { + while (rs.next()) { + return rs.getString(2); + } + return null; + } + }); + // 强制覆盖掉内存值 + tableMetaManager.apply(position, schema, createDDL); + tableMeta = tableMetaManager.find(schema, table); } - } catch (IOException e) { + return tableMeta; + } else { + if (!useCache) { + tableMetaDB.invalidate(getFullName(schema, table)); + } + + return tableMetaDB.getUnchecked(getFullName(schema, table)); } } - public TableMeta getTableMeta(String schema, String table) { - return getTableMeta(schema, table, true); - } - public TableMeta getTableMeta(String schema, String table, boolean useCache) { - if (!useCache) { - tableMetaCache.remove(getFullName(schema, table)); - } - - return tableMetaCache.get(getFullName(schema, table)); - } public void clearTableMeta(String schema, String table) { - tableMetaCache.remove(getFullName(schema, table)); + if (tableMetaManager != null) { + // tsdb不需要做,会基于ddl sql自动清理 + } else { + tableMetaDB.invalidate(getFullName(schema, table)); + } } public void clearTableMetaWithSchemaName(String schema) { - // Set removeNames = new HashSet(); // - // 存一份临时变量，避免在遍历的时候进行删除 - for (String name : tableMetaCache.keySet()) { - if (StringUtils.startsWithIgnoreCase(name, schema + ".")) { - // removeNames.add(name); - tableMetaCache.remove(name); + if (tableMetaManager != null) { + // tsdb不需要做,会基于ddl sql自动清理 + } else { + for (String name : tableMetaDB.asMap().keySet()) { + if (StringUtils.startsWithIgnoreCase(name, schema + ".")) { + // removeNames.add(name); + tableMetaDB.invalidate(name); + } } } - - // for (String name : removeNames) { - // tables.remove(name); - // } } public void clearTableMeta() { - tableMetaCache.clear(); - } - - private TableMeta getTableMeta0(String fullname) throws IOException { - ResultSetPacket packet = connection.query("desc " + fullname); - return new TableMeta(fullname, parserTableMeta(packet)); - } - - private List parserTableMeta(ResultSetPacket packet) { - Map nameMaps = new HashMap(6, 1f); - - int index = 0; - for (FieldPacket fieldPacket : packet.getFieldDescriptors()) { - nameMaps.put(fieldPacket.getOriginalName(), index++); + if (tableMetaManager != null) { + // tsdb不需要做,会基于ddl sql自动清理 + } else { + tableMetaDB.invalidateAll(); } - - int size = packet.getFieldDescriptors().size(); - int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size(); - List result = new ArrayList(); - for (int i = 0; i < count; i++) { - FieldMeta meta = new FieldMeta(); - // 做一个优化，使用String.intern()，共享String对象，减少内存使用 - meta.setColumnName(packet.getFieldValues().get(nameMaps.get(COLUMN_NAME) + i * size).intern()); - meta.setColumnType(packet.getFieldValues().get(nameMaps.get(COLUMN_TYPE) + i * size)); - meta.setIsNullable(packet.getFieldValues().get(nameMaps.get(IS_NULLABLE) + i * size)); - meta.setIskey(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size)); - meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size)); - meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size)); - - result.add(meta); - } - - return result; } + /** + * 更新一下本地的表结构内存 + * + * @param position + * @param schema + * @param ddl + * @return + */ + public boolean apply(BinlogPosition position, String schema, String ddl) { + if (tableMetaManager != null) { + return tableMetaManager.apply(position, schema, ddl); + } else { + // ignore + return true; + } + } + + + private String getFullName(String schema, String table) { StringBuilder builder = new StringBuilder(); return builder.append('`') diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java new file mode 100644 index 00000000..44975ffb --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java @@ -0,0 +1,116 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.ddl; + +import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DBMSAction; + +/** + * @author agapple 2017年8月1日 下午7:30:42 + * @since 3.2.5 + */ +public class DdlResult { + + private String schemaName; + private String tableName; + private String oriSchemaName; // rename ddl中的源表 + private String oriTableName; // rename ddl中的目标表 + private DBMSAction type; + private DdlResult renameTableResult; // 多个rename table的存储 + + /* + * RENAME TABLE tbl_name TO new_tbl_name [, tbl_name2 TO new_tbl_name2] ... + */ + + public DdlResult() { + } + + public DdlResult(String schemaName) { + this.schemaName = schemaName; + } + + public DdlResult(String schemaName, String tableName) { + this.schemaName = schemaName; + this.tableName = tableName; + } + + public DdlResult(String schemaName, String tableName, String oriSchemaName, String oriTableName) { + this.schemaName = schemaName; + this.tableName = tableName; + this.oriSchemaName = oriSchemaName; + this.oriTableName = oriTableName; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public DBMSAction getType() { + return type; + } + + public void setType(DBMSAction type) { + this.type = type; + } + + public String getOriSchemaName() { + return oriSchemaName; + } + + public void setOriSchemaName(String oriSchemaName) { + this.oriSchemaName = oriSchemaName; + } + + public String getOriTableName() { + return oriTableName; + } + + public void setOriTableName(String oriTableName) { + this.oriTableName = oriTableName; + } + + public DdlResult getRenameTableResult() { + return renameTableResult; + } + + public void setRenameTableResult(DdlResult renameTableResult) { + this.renameTableResult = renameTableResult; + } + + @Override + public DdlResult clone() { + DdlResult result = new DdlResult(); + result.setOriSchemaName(oriSchemaName); + result.setOriTableName(oriTableName); + result.setSchemaName(schemaName); + result.setTableName(tableName); + //result.setType(type); + return result; + } + + @Override + public String toString() { + DdlResult ddlResult = this; + StringBuffer sb = new StringBuffer(); + do { + sb.append(String + .format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];", + ddlResult.schemaName, + ddlResult.tableName, + ddlResult.oriSchemaName, + ddlResult.oriTableName, + ddlResult.type)); + ddlResult = ddlResult.renameTableResult; + } while (ddlResult != null); + return sb.toString(); + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java new file mode 100644 index 00000000..647955c5 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java @@ -0,0 +1,201 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.ddl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddConstraint; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddIndex; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropConstraint; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropIndex; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropKey; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableRename; +import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement; +import com.alibaba.druid.sql.ast.statement.SQLConstraint; +import com.alibaba.druid.sql.ast.statement.SQLCreateIndexStatement; +import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; +import com.alibaba.druid.sql.ast.statement.SQLDeleteStatement; +import com.alibaba.druid.sql.ast.statement.SQLDropIndexStatement; +import com.alibaba.druid.sql.ast.statement.SQLDropTableStatement; +import com.alibaba.druid.sql.ast.statement.SQLExprTableSource; +import com.alibaba.druid.sql.ast.statement.SQLInsertStatement; +import com.alibaba.druid.sql.ast.statement.SQLTableSource; +import com.alibaba.druid.sql.ast.statement.SQLTruncateStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnique; +import com.alibaba.druid.sql.ast.statement.SQLUpdateStatement; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement; +import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement.Item; +import com.alibaba.druid.sql.parser.ParserException; +import com.alibaba.druid.util.JdbcConstants; +import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DBMSAction; + +/** + * @author agapple 2017年7月27日 下午4:05:34 + * @since 3.2.5 + */ +public class DruidDdlParser { + + public static List parse(String queryString, String schmeaName) { + List stmtList = null; + try { + stmtList = SQLUtils.parseStatements(queryString, JdbcConstants.MYSQL, false); + } catch (ParserException e) { + // 可能存在一些SQL是不支持的，比如存储过程 + DdlResult ddlResult = new DdlResult(); + ddlResult.setType(DBMSAction.QUERY); + return Arrays.asList(ddlResult); + } + + List ddlResults = new ArrayList(); + for (SQLStatement statement : stmtList) { + if (statement instanceof SQLCreateTableStatement) { + DdlResult ddlResult = new DdlResult(); + SQLCreateTableStatement createTable = (SQLCreateTableStatement) statement; + processName(ddlResult, schmeaName, createTable.getName(), false); + ddlResult.setType(DBMSAction.CREATE); + ddlResults.add(ddlResult); + } else if (statement instanceof SQLAlterTableStatement) { + SQLAlterTableStatement alterTable = (SQLAlterTableStatement) statement; + for (SQLAlterTableItem item : alterTable.getItems()) { + if (item instanceof SQLAlterTableRename) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, alterTable.getName(), true); + processName(ddlResult, schmeaName, ((SQLAlterTableRename) item).getToName(), false); + ddlResults.add(ddlResult); + } else if (item instanceof SQLAlterTableAddIndex) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, alterTable.getName(), false); + ddlResult.setType(DBMSAction.CINDEX); + ddlResults.add(ddlResult); + } else if (item instanceof SQLAlterTableDropIndex || item instanceof SQLAlterTableDropKey) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, alterTable.getName(), false); + ddlResult.setType(DBMSAction.DINDEX); + ddlResults.add(ddlResult); + } else if (item instanceof SQLAlterTableAddConstraint) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, alterTable.getName(), false); + SQLConstraint constraint = ((SQLAlterTableAddConstraint) item).getConstraint(); + if (constraint instanceof SQLUnique) { + ddlResult.setType(DBMSAction.CINDEX); + ddlResults.add(ddlResult); + } + } else if (item instanceof SQLAlterTableDropConstraint) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, alterTable.getName(), false); + ddlResult.setType(DBMSAction.DINDEX); + ddlResults.add(ddlResult); + } else { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, alterTable.getName(), false); + ddlResult.setType(DBMSAction.ALTER); + ddlResults.add(ddlResult); + } + } + } else if (statement instanceof SQLDropTableStatement) { + SQLDropTableStatement dropTable = (SQLDropTableStatement) statement; + for (SQLExprTableSource tableSource : dropTable.getTableSources()) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, tableSource.getExpr(), false); + ddlResult.setType(DBMSAction.ERASE); + ddlResults.add(ddlResult); + } + } else if (statement instanceof SQLCreateIndexStatement) { + SQLCreateIndexStatement createIndex = (SQLCreateIndexStatement) statement; + SQLTableSource tableSource = createIndex.getTable(); + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, ((SQLExprTableSource) tableSource).getExpr(), false); + ddlResult.setType(DBMSAction.CINDEX); + ddlResults.add(ddlResult); + } else if (statement instanceof SQLDropIndexStatement) { + SQLDropIndexStatement dropIndex = (SQLDropIndexStatement) statement; + SQLExprTableSource tableSource = dropIndex.getTableName(); + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, tableSource.getExpr(), false); + ddlResult.setType(DBMSAction.DINDEX); + ddlResults.add(ddlResult); + } else if (statement instanceof SQLTruncateStatement) { + SQLTruncateStatement truncate = (SQLTruncateStatement) statement; + for (SQLExprTableSource tableSource : truncate.getTableSources()) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, tableSource.getExpr(), false); + ddlResult.setType(DBMSAction.TRUNCATE); + ddlResults.add(ddlResult); + } + } else if (statement instanceof MySqlRenameTableStatement) { + MySqlRenameTableStatement rename = (MySqlRenameTableStatement) statement; + for (Item item : rename.getItems()) { + DdlResult ddlResult = new DdlResult(); + processName(ddlResult, schmeaName, item.getName(), true); + processName(ddlResult, schmeaName, item.getTo(), false); + ddlResult.setType(DBMSAction.RENAME); + ddlResults.add(ddlResult); + } + } else if (statement instanceof SQLInsertStatement) { + DdlResult ddlResult = new DdlResult(); + SQLInsertStatement insert = (SQLInsertStatement) statement; + processName(ddlResult, schmeaName, insert.getTableName(), true); + ddlResult.setType(DBMSAction.INSERT); + ddlResults.add(ddlResult); + } else if (statement instanceof SQLUpdateStatement) { + DdlResult ddlResult = new DdlResult(); + SQLUpdateStatement update = (SQLUpdateStatement) statement; + // 拿到的表名可能为null,比如update a,b set a.id=x + processName(ddlResult, schmeaName, update.getTableName(), true); + ddlResult.setType(DBMSAction.UPDATE); + ddlResults.add(ddlResult); + } else if (statement instanceof SQLDeleteStatement) { + DdlResult ddlResult = new DdlResult(); + SQLDeleteStatement delete = (SQLDeleteStatement) statement; + // 拿到的表名可能为null,比如delete a,b from a where a.id = b.id + processName(ddlResult, schmeaName, delete.getTableName(), true); + ddlResult.setType(DBMSAction.DELETE); + ddlResults.add(ddlResult); + } + } + + return ddlResults; + } + + private static void processName(DdlResult ddlResult, String schema, SQLExpr sqlName, boolean isOri) { + if (sqlName == null) { + return; + } + + String table = null; + if (sqlName instanceof SQLPropertyExpr) { + SQLIdentifierExpr owner = (SQLIdentifierExpr) ((SQLPropertyExpr) sqlName).getOwner(); + schema = unescapeName(owner.getName()); + table = unescapeName(((SQLPropertyExpr) sqlName).getName()); + } else if (sqlName instanceof SQLIdentifierExpr) { + table = unescapeName(((SQLIdentifierExpr) sqlName).getName()); + } + + if (isOri) { + ddlResult.setOriSchemaName(schema); + ddlResult.setOriTableName(table); + } else { + ddlResult.setSchemaName(schema); + ddlResult.setTableName(table); + } + } + + public static String unescapeName(String name) { + if (name.length() > 2) { + char c0 = name.charAt(0); + char x0 = name.charAt(name.length() - 1); + if ((c0 == '"' && x0 == '"') || (c0 == '`' && x0 == '`')) { + return name.substring(1, name.length() - 1); + } + } + + return name; + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java new file mode 100644 index 00000000..118da365 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java @@ -0,0 +1,43 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.sql.SQLException; + +import javax.sql.DataSource; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.parse.exception.CanalParseException; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by wanshao + * Date: 2017/9/22 + * Time: 下午2:46 + **/ +public class DataSourceFactoryTSDB { + + private static Logger logger = LoggerFactory.getLogger(DataSourceFactoryTSDB.class); + + public static DataSource getDataSource(String address, String userName, String password, + boolean enable, + String defaultDatabaseName, String url, String driverClassName) { + + DruidDataSource druidDataSource = new DruidDataSource(); + druidDataSource.setUrl(url); + druidDataSource.setUsername(userName); + druidDataSource.setPassword(password); + + try { + druidDataSource.init(); + } catch (SQLException e) { + logger.error("druidDataSource.init", e); + throw new CanalParseException("初始化druid dataSource出错"); + } + return druidDataSource; + } + + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java new file mode 100644 index 00000000..293661f4 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -0,0 +1,223 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.taobao.tddl.dbsync.binlog.BinlogPosition; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.druid.sql.ast.SQLDataType; +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLNullExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLColumnConstraint; +import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition; +import com.alibaba.druid.sql.ast.statement.SQLColumnPrimaryKey; +import com.alibaba.druid.sql.ast.statement.SQLCreateTableStatement; +import com.alibaba.druid.sql.ast.statement.SQLNotNullConstraint; +import com.alibaba.druid.sql.ast.statement.SQLNullConstraint; +import com.alibaba.druid.sql.ast.statement.SQLSelectOrderByItem; +import com.alibaba.druid.sql.ast.statement.SQLTableElement; +import com.alibaba.druid.sql.dialect.mysql.ast.MySqlPrimaryKey; +import com.alibaba.druid.sql.repository.Schema; +import com.alibaba.druid.sql.repository.SchemaObject; +import com.alibaba.druid.sql.repository.SchemaRepository; +import com.alibaba.druid.util.JdbcConstants; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; + +/** + * 基于DDL维护的内存表结构 + * + * @author agapple 2017年7月27日 下午4:19:40 + * @since 3.2.5 + */ +public class MemoryTableMeta implements TableMetaTSDB { + + private Map, TableMeta> tableMetas = new ConcurrentHashMap, TableMeta>(); + private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); + private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class); + + public MemoryTableMeta(Logger logger){ + this.logger = logger; + } + + public boolean apply(BinlogPosition position, String schema, String ddl) { + tableMetas.clear(); + synchronized (this) { + if (StringUtils.isNotEmpty(schema)) { + repository.setDefaultSchema(schema); + } + + try { + repository.console(ddl); + } catch (Throwable e) { + logger.warn("parse faield : " + ddl, e); + } + } + + // TableMeta meta = find("tddl5_00", "ab"); + // if (meta != null) { + // repository.setDefaultSchema("tddl5_00"); + // System.out.println(repository.console("show create table tddl5_00.ab")); + // System.out.println(repository.console("show columns from tddl5_00.ab")); + // } + return true; + } + + @Override + public TableMeta find(String schema, String table) { + List keys = Arrays.asList(schema, table); + TableMeta tableMeta = tableMetas.get(keys); + if (tableMeta == null) { + synchronized (this) { + tableMeta = tableMetas.get(keys); + if (tableMeta == null) { + Schema schemaRep = repository.findSchema(schema); + if (schema == null) { + return null; + } + SchemaObject data = schemaRep.findTable(table); + if (data == null) { + return null; + } + SQLStatement statement = data.getStatement(); + if (statement == null) { + return null; + } + if (statement instanceof SQLCreateTableStatement) { + tableMeta = parse((SQLCreateTableStatement) statement); + } + if (tableMeta != null) { + if (table != null) { + tableMeta.setTable(table); + } + if (schema != null) { + tableMeta.setSchema(schema); + } + + tableMetas.put(keys, tableMeta); + } + } + } + } + + return tableMeta; + } + + @Override + public boolean rollback(BinlogPosition position) { + throw new RuntimeException("not support for memory"); + } + + public Map snapshot() { + Map schemaDdls = new HashMap(); + for (Schema schema : repository.getSchemas()) { + StringBuffer data = new StringBuffer(4 * 1024); + for (String table : schema.showTables()) { + SchemaObject schemaObject = schema.findTable(table); + schemaObject.getStatement().output(data); + data.append("; \n"); + } + schemaDdls.put(schema.getName(), data.toString()); + } + + return schemaDdls; + } + + private TableMeta parse(SQLCreateTableStatement statement) { + int size = statement.getTableElementList().size(); + if (size > 0) { + TableMeta tableMeta = new TableMeta(); + for (int i = 0; i < size; ++i) { + SQLTableElement element = statement.getTableElementList().get(i); + processTableElement(element, tableMeta); + } + return tableMeta; + } + + return null; + } + + private void processTableElement(SQLTableElement element, TableMeta tableMeta) { + if (element instanceof SQLColumnDefinition) { + FieldMeta fieldMeta = new FieldMeta(); + SQLColumnDefinition column = (SQLColumnDefinition) element; + String name = getSqlName(column.getName()); + // String charset = getSqlName(column.getCharsetExpr()); + SQLDataType dataType = column.getDataType(); + String dataTypStr = dataType.getName(); + if (dataType.getArguments().size() > 0) { + dataTypStr += "("; + for (int i = 0; i < column.getDataType().getArguments().size(); i++) { + if (i != 0) { + dataTypStr += ","; + } + SQLExpr arg = column.getDataType().getArguments().get(i); + dataTypStr += arg.toString(); + } + dataTypStr += ")"; + } + + if (column.getDefaultExpr() == null || column.getDefaultExpr() instanceof SQLNullExpr) { + fieldMeta.setDefaultValue(null); + } else { + fieldMeta.setDefaultValue(getSqlName(column.getDefaultExpr())); + } + + fieldMeta.setColumnName(name); + fieldMeta.setColumnType(dataTypStr); + fieldMeta.setNullable(true); + List constraints = column.getConstraints(); + for (SQLColumnConstraint constraint : constraints) { + if (constraint instanceof SQLNotNullConstraint) { + fieldMeta.setNullable(false); + } else if (constraint instanceof SQLNullConstraint) { + fieldMeta.setNullable(true); + } else if (constraint instanceof SQLColumnPrimaryKey) { + fieldMeta.setKey(true); + } + } + tableMeta.addFieldMeta(fieldMeta); + } else if (element instanceof MySqlPrimaryKey) { + MySqlPrimaryKey column = (MySqlPrimaryKey) element; + List pks = column.getColumns(); + for (SQLSelectOrderByItem pk : pks) { + String name = getSqlName(pk.getExpr()); + FieldMeta field = tableMeta.getFieldMetaByName(name); + field.setKey(true); + } + } + } + + private String getSqlName(SQLExpr sqlName) { + if (sqlName == null) { + return null; + } + + if (sqlName instanceof SQLPropertyExpr) { + SQLIdentifierExpr owner = (SQLIdentifierExpr) ((SQLPropertyExpr) sqlName).getOwner(); + return DruidDdlParser.unescapeName(owner.getName()) + "." + + DruidDdlParser.unescapeName(((SQLPropertyExpr) sqlName).getName()); + } else if (sqlName instanceof SQLIdentifierExpr) { + return DruidDdlParser.unescapeName(((SQLIdentifierExpr) sqlName).getName()); + } else { + return sqlName.toString(); + } + } + + public SchemaRepository getRepository() { + return repository; + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java new file mode 100644 index 00000000..859152f7 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java @@ -0,0 +1,565 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import javax.annotation.Resource; + +import com.alibaba.druid.sql.repository.Schema; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.otter.canal.filter.CanalEventFilter; +import com.alibaba.otter.canal.parse.exception.CanalParseException; +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.ProcessJdbcResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; + +import com.taobao.tddl.dbsync.binlog.BinlogPosition; +import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.lang.ObjectUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 基于console远程管理 + * + * see internal class: CanalTableMeta , ConsoleTableMetaTSDB + * + * @author agapple 2017年7月27日 下午10:47:55 + * @since 3.2.5 + */ +public class TableMetaManager implements TableMetaTSDB { + private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); + + private static final BinlogPosition INIT_POSITION = BinlogPosition.parseFromString("0:0#-2.-1"); + private Logger logger = LoggerFactory.getLogger(TableMetaManager.class); + private String consoleDomain = null; + private int retry = 3; + private MemoryTableMeta memoryTableMeta; + private MysqlConnection connection; // 查询meta信息的链接 + private CanalEventFilter filter; + private BinlogPosition lastPosition; + private ScheduledExecutorService scheduler; + + @Resource + private MetaHistoryDAO metaHistoryDAO; + + @Resource + private MetaSnapshotDAO metaSnapshotDAO; + + public void init(){ + this.memoryTableMeta = new MemoryTableMeta(logger); + this.scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "[scheduler-table-meta-snapshot]"); + } + }); + + // 24小时生成一份snapshot + scheduler.scheduleWithFixedDelay(new Runnable() { + + @Override + public void run() { + try { + logger.info("-------- begin to produce snapshot for table meta"); + applySnapshotToDB(lastPosition, false); + } catch (Throwable e) { + logger.error("scheudle faield", e); + } + } + }, 24, 24, TimeUnit.SECONDS); + } + + public TableMetaManager() { + + } + + @Override + public TableMeta find(String schema, String table) { + synchronized (memoryTableMeta) { + return memoryTableMeta.find(schema, table); + } + } + + @Override + public boolean apply(BinlogPosition position, String schema, String ddl) { + // 首先记录到内存结构 + synchronized (memoryTableMeta) { + if (memoryTableMeta.apply(position, schema, ddl)) { + this.lastPosition = position; + // 同步每次变更给远程做历史记录 + return applyHistoryToDB(position, schema, ddl); + } else { + throw new RuntimeException("apply to memory is failed"); + } + } + } + + @Override + public boolean rollback(BinlogPosition position) { + // 每次rollback需要重新构建一次memory data + this.memoryTableMeta = new MemoryTableMeta(logger); + boolean flag = false; + BinlogPosition snapshotPosition = buildMemFromSnapshot(position); + if (snapshotPosition != null) { + applyHistoryOnMemory(snapshotPosition, position); + flag = true; + } + + if (!flag) { + // 如果没有任何数据，则为初始化状态，全量dump一份关注的表 + if (dumpTableMeta(connection, filter)) { + // 记录一下snapshot结果,方便快速恢复 + flag = applySnapshotToDB(INIT_POSITION, true); + } + } + + return flag; + } + + @Override + public Map snapshot() { + return memoryTableMeta.snapshot(); + } + + /** + * 初始化的时候dump一下表结构 + */ + private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter filter) { + List schemas = connection.query("show databases", new ProcessJdbcResult() { + + @Override + public List process(ResultSet rs) throws SQLException { + List schemas = new ArrayList(); + while (rs.next()) { + String schema = rs.getString(1); + if (!filter.filter(schema)) { + schemas.add(schema); + } + } + return schemas; + } + }); + + for (String schema : schemas) { + List tables = connection.query("show tables from `" + schema + "`", new ProcessJdbcResult() { + + @Override + public List process(ResultSet rs) throws SQLException { + List tables = new ArrayList(); + while (rs.next()) { + String table = rs.getString(1); + if (!filter.filter(table)) { + tables.add(table); + } + } + return tables; + } + }); + + StringBuilder sql = new StringBuilder(); + for (String table : tables) { + sql.append("show create table `" + schema + "`.`" + table + "`;"); + } + + // 使用多语句方式读取 + Statement stmt = null; + try { + stmt = connection.getConn().createStatement(); + ResultSet rs = stmt.executeQuery(sql.toString()); + boolean existMoreResult = false; + do { + if (existMoreResult) { + rs = stmt.getResultSet(); + } + + while (rs.next()) { + String oneTableCreateSql = rs.getString(2); + memoryTableMeta.apply(INIT_POSITION, schema, oneTableCreateSql); + } + + existMoreResult = stmt.getMoreResults(); + } while (existMoreResult); + } catch (SQLException e) { + throw new CanalParseException(e); + } finally { + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + // ignore + } + } + } + } + + return true; + } + + private boolean applyHistoryToDB(BinlogPosition position, String schema, String ddl) { + + Map content = new HashMap(); + + content.put("binlogFile", position.getFileName()); + content.put("binlogOffest", String.valueOf(position.getPosition())); + content.put("binlogMasterId", String.valueOf(position.getMasterId())); + content.put("binlogTimestamp", String.valueOf(position.getTimestamp())); + content.put("useSchema", schema); + if (content.isEmpty()) { + throw new RuntimeException("apply failed caused by content is empty in applyHistoryToDB"); + } + // 待补充 + List ddlResults = DruidDdlParser.parse(schema, ddl); + if (ddlResults.size() > 0) { + DdlResult ddlResult = ddlResults.get(0); + content.put("schema", ddlResult.getSchemaName()); + content.put("table", ddlResult.getTableName()); + content.put("type", ddlResult.getType().name()); + content.put("sql", ddl); + // content.put("extra", ""); + } + + for (int i = 0; i < retry; i++) { + MetaHistoryDO metaDO = new MetaHistoryDO(); + try { + BeanUtils.populate(metaDO, content); + // 会建立唯一约束,解决: + // 1. 重复的binlog file+offest + // 2. 重复的masterId+timestamp + metaHistoryDAO.insert(metaDO); + } catch (Throwable e) { + if (isUkDuplicateException(e)) { + // 忽略掉重复的位点 + logger.warn("dup apply for sql : " + ddl); + } else { + throw new RuntimeException("apply history to db failed caused by : " + e.getMessage()); + } + + } + return true; + } + return false; + } + + /** + * 发布数据到console上 + */ + private boolean applySnapshotToDB(BinlogPosition position, boolean init) { + // 获取一份快照 + MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta(logger); + Map schemaDdls = null; + synchronized (memoryTableMeta) { + if (!init && position == null) { + // 如果是持续构建,则识别一下是否有DDL变更过,如果没有就忽略了 + return false; + } + schemaDdls = memoryTableMeta.snapshot(); + for (Map.Entry entry : schemaDdls.entrySet()) { + tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue()); + } + } + + // 基于临时内存对象进行对比 + boolean compareAll = true; + for (Schema schema : tmpMemoryTableMeta.getRepository().getSchemas()) { + for (String table : schema.showTables()) { + if (!compareTableMetaDbAndMemory(connection, schema.getName(), table)) { + compareAll = false; + } + } + } + if (compareAll) { + + Map content = new HashMap(); + + content.put("binlogFile", position.getFileName()); + content.put("binlogOffest", String.valueOf(position.getPosition())); + content.put("binlogMasterId", String.valueOf(position.getMasterId())); + content.put("binlogTimestamp", String.valueOf(position.getTimestamp())); + content.put("data", JSON.toJSONString(schemaDdls)); + if (content.isEmpty()) { + throw new RuntimeException("apply failed caused by content is empty in applySnapshotToDB"); + } + + for (int i = 0; i < retry; i++) { + MetaSnapshotDO snapshotDO = new MetaSnapshotDO(); + try { + BeanUtils.populate(snapshotDO, content); + metaSnapshotDAO.insert(snapshotDO); + } catch (Throwable e) { + if (isUkDuplicateException(e)) { + // 忽略掉重复的位点 + logger.warn("dup apply snapshot for data : " + snapshotDO.getData()); + } else { + throw new RuntimeException("apply failed caused by : " + e.getMessage()); + } + } + return true; + + } + return false; + + } else { + logger.error("compare failed , check log"); + } + return false; + } + + private boolean compareTableMetaDbAndMemory(MysqlConnection connection, final String schema, final String table) { + TableMeta tableMetaFromDB = connection.query("desc " + getFullName(schema, table), + new ProcessJdbcResult() { + + @Override + public TableMeta process(ResultSet rs) throws SQLException { + List metas = new ArrayList(); + while (rs.next()) { + FieldMeta meta = new FieldMeta(); + // 做一个优化，使用String.intern()，共享String对象，减少内存使用 + meta.setColumnName(rs.getString("Field")); + meta.setColumnType(rs.getString("Type")); + meta.setNullable(StringUtils.equalsIgnoreCase(rs.getString("Null"), "YES")); + meta.setKey("PRI".equalsIgnoreCase(rs.getString("Key"))); + meta.setDefaultValue(rs.getString("Default")); + metas.add(meta); + } + + return new TableMeta(schema, table, metas); + } + }); + + TableMeta tableMetaFromMem = memoryTableMeta.find(schema, table); + boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB); + if (!result) { + logger.error("compare failed . \n db : " + tableMetaFromDB + " \n mem : " + tableMetaFromMem); + } + + return result; + } + + private BinlogPosition buildMemFromSnapshot(BinlogPosition position) { + + Map content = new HashMap(); + + content.put("binlogFile", position.getFileName()); + content.put("binlogOffest", String.valueOf(position.getPosition())); + content.put("binlogMasterId", String.valueOf(position.getMasterId())); + content.put("binlogTimestamp", String.valueOf(position.getTimestamp())); + if (content.isEmpty()) { + throw new RuntimeException("apply failed caused by content is empty in buildMemFromSnapshot"); + } + for (int i = 0; i < retry; i++) { + + try { + + String timestamp = content.get("binlogTimestamp"); + MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(Long.valueOf(timestamp)); + JSONObject jsonData = new JSONObject(); + jsonData.put("content", JSON.toJSONString(snapshotDO)); + if (jsonData == null) { + // 可能没有任何snapshot数据 + return null; + } + + String binlogFile = jsonData.getString("binlogFile"); + String binlogOffest = jsonData.getString("binlogOffest"); + String binlogMasterId = jsonData.getString("binlogMasterId"); + String binlogTimestamp = jsonData.getString("binlogTimestamp"); + + BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, + Long.valueOf(binlogOffest == null ? "0" : binlogOffest), + Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), + Long.valueOf(binlogTimestamp == null ? "0" : binlogTimestamp)); + // data存储为Map，每个分库一套建表 + String sqlData = jsonData.getString("data"); + JSONObject jsonObj = JSON.parseObject(sqlData); + for (Map.Entry entry : jsonObj.entrySet()) { + // 记录到内存 + if (!memoryTableMeta.apply(snapshotPosition, + ObjectUtils.toString(entry.getKey()), + ObjectUtils.toString(entry.getValue()))) { + return null; + } + } + + return snapshotPosition; + + } catch (Throwable e) { + throw new RuntimeException("apply failed caused by : " + e.getMessage()); + } + + } + + return null; + + } + + private boolean applyHistoryOnMemory(BinlogPosition position, BinlogPosition rollbackPosition) { + Map content = new HashMap(); + content.put("binlogSnapshotTimestamp", String.valueOf(position.getTimestamp())); + content.put("binlogFile", rollbackPosition.getFileName()); + content.put("binlogOffest", String.valueOf(rollbackPosition.getPosition())); + content.put("binlogMasterId", String.valueOf(rollbackPosition.getMasterId())); + content.put("binlogTimestamp", String.valueOf(rollbackPosition.getTimestamp())); + String timestamp = content.get("binlogTimestamp"); + String binlogSnapshotTimestamp = content.get("binlogSnapshotTimestamp"); + + for (int i = 0; i < retry; i++) { + try { + List metaHistoryDOList = metaHistoryDAO.findByTimestamp( + Long.valueOf(binlogSnapshotTimestamp), + Long.valueOf(timestamp)); + JSONObject json = new JSONObject(); + json.put("content", JSON.toJSONString(metaHistoryDOList)); + String data = ObjectUtils.toString(json.get("content")); + JSONArray jsonArray = JSON.parseArray(data); + for (Object jsonObj : jsonArray) { + JSONObject jsonData = (JSONObject)jsonObj; + String binlogFile = jsonData.getString("binlogFile"); + String binlogOffest = jsonData.getString("binlogOffest"); + String binlogMasterId = jsonData.getString("binlogMasterId"); + String binlogTimestamp = jsonData.getString("binlogTimestamp"); + String useSchema = jsonData.getString("useSchema"); + String sqlData = jsonData.getString("sql"); + BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, + Long.valueOf(binlogOffest == null ? "0" : binlogOffest), + Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), + Long.valueOf(binlogTimestamp == null ? "0" : binlogTimestamp)); + + // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之 + if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) { + continue; + } else if (rollbackPosition.getMasterId() == snapshotPosition.getMasterId() + && snapshotPosition.compareTo(rollbackPosition) > 0) { + continue; + } + + // 记录到内存 + if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData)) { + return false; + } + + } + + return jsonArray.size() > 0; + } catch (Throwable e) { + + throw new RuntimeException("apply failed", e); + + } + + } + + return false; + } + + private String getFullName(String schema, String table) { + StringBuilder builder = new StringBuilder(); + return builder.append('`') + .append(schema) + .append('`') + .append('.') + .append('`') + .append(table) + .append('`') + .toString(); + } + + private boolean compareTableMeta(TableMeta source, TableMeta target) { + if (!StringUtils.equalsIgnoreCase(source.getSchema(), target.getSchema())) { + return false; + } + + if (!StringUtils.equalsIgnoreCase(source.getTable(), target.getTable())) { + return false; + } + + List sourceFields = source.getFields(); + List targetFields = target.getFields(); + if (sourceFields.size() != targetFields.size()) { + return false; + } + + for (int i = 0; i < sourceFields.size(); i++) { + FieldMeta sourceField = sourceFields.get(i); + FieldMeta targetField = targetFields.get(i); + if (!StringUtils.equalsIgnoreCase(sourceField.getColumnName(), targetField.getColumnName())) { + return false; + } + + if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(), targetField.getColumnType())) { + return false; + } + + if (!StringUtils.equalsIgnoreCase(sourceField.getDefaultValue(), targetField.getDefaultValue())) { + return false; + } + + if (sourceField.isNullable() != targetField.isNullable()) { + return false; + } + + if (sourceField.isKey() != targetField.isKey()) { + return false; + } + } + + return true; + } + + public void setConnection(MysqlConnection connection) { + this.connection = connection; + } + + public void setFilter(CanalEventFilter filter) { + this.filter = filter; + } + + public MetaHistoryDAO getMetaHistoryDAO() { + return metaHistoryDAO; + } + + public void setMetaHistoryDAO(MetaHistoryDAO metaHistoryDAO) { + this.metaHistoryDAO = metaHistoryDAO; + } + + public MetaSnapshotDAO getMetaSnapshotDAO() { + return metaSnapshotDAO; + } + + public void setMetaSnapshotDAO(MetaSnapshotDAO metaSnapshotDAO) { + this.metaSnapshotDAO = metaSnapshotDAO; + } + + public MysqlConnection getConnection() { + return connection; + } + + public boolean isUkDuplicateException(Throwable t) { + if (pattern.matcher(t.getMessage()).find()) { + // 违反外键约束时也抛出这种异常，所以这里还要判断包含字符串Duplicate entry + return true; + } + return false; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java new file mode 100644 index 00000000..44cd9959 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java @@ -0,0 +1,37 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.util.Map; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; + +import com.taobao.tddl.dbsync.binlog.BinlogPosition; + +/** + * 表结构的时间序列存储 + * + * @author agapple 2017年7月27日 下午4:06:30 + * @since 3.2.5 + */ +public interface TableMetaTSDB { + + /** + * 获取当前的表结构 + */ + public TableMeta find(String schema, String table); + + /** + * 添加ddl到时间序列库中 + */ + public boolean apply(BinlogPosition position, String schema, String ddl); + + /** + * 回滚到指定位点的表结构 + */ + public boolean rollback(BinlogPosition position); + + /** + * 生成快照内容 + */ + public Map snapshot(); + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java new file mode 100644 index 00000000..054a4b44 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java @@ -0,0 +1,44 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; + +import java.util.HashMap; +import java.util.List; + +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; + +import com.google.common.collect.Maps; +import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; + +/** + * canal数据的存储 + * + * @author wanshao 2017年7月27日 下午10:51:55 + * @since 3.2.5 + */ +@SuppressWarnings("deprecation") +public class MetaHistoryDAO extends SqlMapClientDaoSupport { + + public List getAll() { + return getSqlMapClientTemplate().queryForList("table_meta_history.getAll"); + } + + public Long insert(MetaHistoryDO metaDO) { + return (Long)getSqlMapClientTemplate().insert("table_meta_history.insert", metaDO); + } + + public List findByTimestamp(long snapshotTimestamp, long timestamp) { + HashMap params = Maps.newHashMapWithExpectedSize(2); + params.put("snapshotTimestamp", snapshotTimestamp); + params.put("timestamp", timestamp); + return (List)getSqlMapClientTemplate().queryForList("table_meta_history.findByTimestamp", + params); + } + + /** + * 删除interval秒之前的数据 + */ + public Integer deleteByGmtModified(int interval) { + HashMap params = Maps.newHashMapWithExpectedSize(2); + params.put("interval", interval); + return getSqlMapClientTemplate().delete("table_meta_history.deleteByGmtModified", params); + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java new file mode 100644 index 00000000..79f1f16f --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java @@ -0,0 +1,48 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; + +import java.util.HashMap; + +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; + +import com.google.common.collect.Maps; +import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; + +/** + * canal数据的存储 + * + * @author wanshao 2017年7月27日 下午10:51:55 + * @since 3.2.5 + */ + +public class MetaSnapshotDAO extends SqlMapClientDaoSupport { + public Long insert(MetaSnapshotDO snapshotDO) { + return (Long) getSqlMapClientTemplate().insert("table_meta_snapshot.insert", snapshotDO); + } + + public Long update(MetaSnapshotDO snapshotDO) { + return (Long) getSqlMapClientTemplate().insert("table_meta_snapshot.update", snapshotDO); + } + + public MetaSnapshotDO findByTimestamp(long timestamp) { + HashMap params = Maps.newHashMapWithExpectedSize(2); + params.put("timestamp", timestamp); + return (MetaSnapshotDO) getSqlMapClientTemplate().queryForObject("table_meta_snapshot.findByTimestamp", + params); + } + + public Integer deleteByTask(String taskName) { + HashMap params = Maps.newHashMapWithExpectedSize(2); + params.put("taskName", taskName); + return getSqlMapClientTemplate().delete("table_meta_snapshot.deleteByTaskName", params); + } + + /** + * 删除interval秒之前的数据 + */ + public Integer deleteByGmtModified(int interval) { + HashMap params = Maps.newHashMapWithExpectedSize(2); + params.put("interval", interval); + return getSqlMapClientTemplate().delete("table_meta_snapshot.deleteByGmtModified", params); + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java new file mode 100644 index 00000000..9b56606d --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java @@ -0,0 +1,141 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model; + +import java.util.Date; + +/** + * @author agapple 2017年7月27日 下午11:09:41 + * @since 3.2.5 + */ +public class MetaHistoryDO { + + /** + * 主键 + */ + private Long id; + + /** + * 创建时间 + */ + private Date gmtCreate; + + /** + * 修改时间 + */ + private Date gmtModified; + + private String binlogFile; + private Long binlogOffest; + private String binlogMasterId; + private Long binlogTimestamp; + private String useSchema; + private String schema; + private String table; + private String sql; + private String type; + private String extra; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Date getGmtCreate() { + return gmtCreate; + } + + public void setGmtCreate(Date gmtCreate) { + this.gmtCreate = gmtCreate; + } + + public Date getGmtModified() { + return gmtModified; + } + + public void setGmtModified(Date gmtModified) { + this.gmtModified = gmtModified; + } + + public String getBinlogFile() { + return binlogFile; + } + + public void setBinlogFile(String binlogFile) { + this.binlogFile = binlogFile; + } + + public Long getBinlogOffest() { + return binlogOffest; + } + + public void setBinlogOffest(Long binlogOffest) { + this.binlogOffest = binlogOffest; + } + + public String getBinlogMasterId() { + return binlogMasterId; + } + + public void setBinlogMasterId(String binlogMasterId) { + this.binlogMasterId = binlogMasterId; + } + + public Long getBinlogTimestamp() { + return binlogTimestamp; + } + + public void setBinlogTimestamp(Long binlogTimestamp) { + this.binlogTimestamp = binlogTimestamp; + } + + public String getUseSchema() { + return useSchema; + } + + public void setUseSchema(String useSchema) { + this.useSchema = useSchema; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getExtra() { + return extra; + } + + public void setExtra(String extra) { + this.extra = extra; + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java new file mode 100644 index 00000000..4652fe13 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java @@ -0,0 +1,105 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model; + +import java.util.Date; + +/** + * @author agapple 2017年7月27日 下午11:09:41 + * @since 3.2.5 + */ +public class MetaSnapshotDO { + + /** + * 主键 + */ + private Long id; + + /** + * 创建时间 + */ + private Date gmtCreate; + + /** + * 修改时间 + */ + private Date gmtModified; + + private String binlogFile; + private Long binlogOffest; + private String binlogMasterId; + private Long binlogTimestamp; + private String data; + private String extra; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Date getGmtCreate() { + return gmtCreate; + } + + public void setGmtCreate(Date gmtCreate) { + this.gmtCreate = gmtCreate; + } + + public Date getGmtModified() { + return gmtModified; + } + + public void setGmtModified(Date gmtModified) { + this.gmtModified = gmtModified; + } + + public String getBinlogFile() { + return binlogFile; + } + + public void setBinlogFile(String binlogFile) { + this.binlogFile = binlogFile; + } + + public Long getBinlogOffest() { + return binlogOffest; + } + + public void setBinlogOffest(Long binlogOffest) { + this.binlogOffest = binlogOffest; + } + + public String getBinlogMasterId() { + return binlogMasterId; + } + + public void setBinlogMasterId(String binlogMasterId) { + this.binlogMasterId = binlogMasterId; + } + + public Long getBinlogTimestamp() { + return binlogTimestamp; + } + + public void setBinlogTimestamp(Long binlogTimestamp) { + this.binlogTimestamp = binlogTimestamp; + } + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + + public String getExtra() { + return extra; + } + + public void setExtra(String extra) { + this.extra = extra; + } + +} diff --git a/parse/src/main/resources/dal-dao.xml b/parse/src/main/resources/dal-dao.xml new file mode 100644 index 00000000..942de654 --- /dev/null +++ b/parse/src/main/resources/dal-dao.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/parse/src/main/resources/mybatis-config.xml b/parse/src/main/resources/mybatis-config.xml new file mode 100644 index 00000000..83ac0116 --- /dev/null +++ b/parse/src/main/resources/mybatis-config.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/parse/src/main/resources/sql-map/sqlmap_history.xml b/parse/src/main/resources/sql-map/sqlmap_history.xml new file mode 100644 index 00000000..00197194 --- /dev/null +++ b/parse/src/main/resources/sql-map/sqlmap_history.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + insert into `canal_table_meta_history` () + values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#schema#,#table#,#sql#,#type#,#extra#); + + SELECT last_insert_id() + + + + + + + + + + + + \ No newline at end of file diff --git a/parse/src/main/resources/sql-map/sqlmap_snapshot.xml b/parse/src/main/resources/sql-map/sqlmap_snapshot.xml new file mode 100644 index 00000000..9065a7e0 --- /dev/null +++ b/parse/src/main/resources/sql-map/sqlmap_snapshot.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + insert into `canal_table_meta_snapshot` () + values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#); + + SELECT last_insert_id() + + + + + update `canal_table_meta_snapshot` set gmt_modified=now(), + binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra# + where binlog_timestamp=0 + + + + + + + + + + \ No newline at end of file diff --git a/parse/src/main/resources/sqlmap-config.xml b/parse/src/main/resources/sqlmap-config.xml new file mode 100644 index 00000000..983e9663 --- /dev/null +++ b/parse/src/main/resources/sqlmap-config.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java index b0af1256..0c09dc0b 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java @@ -3,31 +3,33 @@ package com.alibaba.otter.canal.parse.inbound; import java.io.IOException; import java.net.InetSocketAddress; +import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.junit.Assert; import org.junit.Test; +import com.alibaba.druid.sql.visitor.functions.Bin; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; public class TableMetaCacheTest { - @Test - public void testSimple() { - - MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"); - try { - connection.connect(); - } catch (IOException e) { - Assert.fail(e.getMessage()); - } - - TableMetaCache cache = new TableMetaCache(connection); - TableMeta meta = cache.getTableMeta("otter1", "otter_stability1"); - Assert.assertNotNull(meta); - for (FieldMeta field : meta.getFileds()) { - System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : " - + field.isNullable()); - } - } + //@Test + //public void testSimple() { + // + // MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"); + // try { + // connection.connect(); + // } catch (IOException e) { + // Assert.fail(e.getMessage()); + // } + // + // TableMetaCache cache = new TableMetaCache(connection); + // TableMeta meta = cache.getTableMeta("otter1", "otter_stability1"); + // Assert.assertNotNull(meta); + // for (FieldMeta field : meta.getFields()) { + // System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : " + // + field.isNullable()); + // } + //} } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java index 3f6dff3c..5d3e8920 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java @@ -88,8 +88,9 @@ public class GroupEventPaserTest { private BinlogParser buildParser(AuthenticationInfo info) { return new AbstractBinlogParser() { - public Entry parse(LogEvent event) throws CanalParseException { - // return _parser.parse(event); + + @Override + public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException { return null; } }; diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java index 212f3af6..a4c1c6df 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java @@ -25,11 +25,11 @@ public class LocalBinlogDumpTest { @Test public void testSimple() { - String directory = "/home/jianghang/tmp/binlog"; + String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb"; final LocalBinlogEventParser controller = new LocalBinlogEventParser(); - final EntryPosition startPosition = new EntryPosition("mysql-bin.000006", 4L); + final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L); - controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx")); + controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal")); controller.setConnectionCharset(Charset.forName("UTF-8")); controller.setDirectory(directory); controller.setMasterPosition(startPosition); diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java index d619916a..7b996ba2 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java @@ -23,15 +23,15 @@ import com.alibaba.otter.canal.sink.exception.CanalSinkException; public class LocalBinlogEventParserTest { private static final String MYSQL_ADDRESS = "127.0.0.1"; - private static final String USERNAME = "xxxxx"; - private static final String PASSWORD = "xxxxx"; + private static final String USERNAME = "canal"; + private static final String PASSWORD = "canal"; private String directory; @Before public void setUp() { URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); File dummyFile = new File(url.getFile()); - directory = new File(dummyFile.getParent() + "/binlog").getPath(); + directory = new File("/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb").getPath(); } @Test @@ -40,7 +40,7 @@ public class LocalBinlogEventParserTest { final AtomicLong entryCount = new AtomicLong(0); final EntryPosition entryPosition = new EntryPosition(); - final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L); + final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", 219L, 1505467103000L); final LocalBinlogEventParser controller = new LocalBinlogEventParser(); controller.setMasterPosition(defaultPosition); controller.setMasterInfo(buildAuthentication()); @@ -103,7 +103,7 @@ public class LocalBinlogEventParserTest { final AtomicLong entryCount = new AtomicLong(0); final EntryPosition entryPosition = new EntryPosition(); - final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, 1322803601000L); + final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", null, 1505467103000L); final LocalBinlogEventParser controller = new LocalBinlogEventParser(); controller.setMasterPosition(defaultPosition); controller.setMasterInfo(buildAuthentication()); @@ -156,15 +156,15 @@ public class LocalBinlogEventParserTest { Assert.assertTrue(entryCount.get() > 0); // 对比第一条数据和起始的position相同 - Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001"); - Assert.assertTrue(entryPosition.getPosition() <= 6163L); + Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000003"); + Assert.assertTrue(entryPosition.getPosition() <= 300L); Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp()); } @Test public void test_no_position() throws InterruptedException { final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000); - final EntryPosition defaultPosition = buildPosition("mysql-bin.000002", + final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", null, new Date().getTime() + 1000 * 1000L); final AtomicLong entryCount = new AtomicLong(0); diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java index f7dab29a..d8915e64 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java @@ -26,12 +26,12 @@ public class MysqlDumpTest { @Test public void testSimple() { final MysqlEventParser controller = new MysqlEventParser(); - final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L); + final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L); controller.setConnectionCharset(Charset.forName("UTF-8")); controller.setSlaveId(3344L); controller.setDetectingEnable(false); - controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx")); + controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal")); controller.setMasterPosition(startPosition); controller.setEventSink(new AbstractCanalEventSinkTest>() { diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java index 82ecf4af..f254c17c 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java @@ -24,8 +24,8 @@ public class MysqlEventParserTest { private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()"; private static final String MYSQL_ADDRESS = "127.0.0.1"; - private static final String USERNAME = "root"; - private static final String PASSWORD = "xxxxxx"; + private static final String USERNAME = "canal"; + private static final String PASSWORD = "canal"; @Test public void test_position() throws InterruptedException { @@ -34,7 +34,7 @@ public class MysqlEventParserTest { final EntryPosition entryPosition = new EntryPosition(); final MysqlEventParser controller = new MysqlEventParser(); - final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L); + final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", 4690L, 1505481064000L); controller.setSlaveId(3344L); controller.setDetectingEnable(true); diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java new file mode 100644 index 00000000..c928b667 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java @@ -0,0 +1,38 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.io.File; +import java.io.FileInputStream; +import java.net.URL; + +import com.alibaba.otter.canal.parse.inbound.TableMeta; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author agapple 2017年8月1日 下午7:15:54 + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = {"/dal-dao.xml"}) +public class MemoryTableMetaTest { + + @Test + public void testSimple() throws Throwable { + MemoryTableMeta memoryTableMeta = new MemoryTableMeta(null); + URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); + File dummyFile = new File(url.getFile()); + File create = new File(dummyFile.getParent() + "/ddl", "create.sql"); + String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); + memoryTableMeta.apply(null, "test", sql); + + TableMeta meta = memoryTableMeta.find("test", "test"); + System.out.println(meta); + Assert.assertTrue( meta.getFieldMetaByName("ID").isKey()); + } +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java new file mode 100644 index 00000000..30ae1e66 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java @@ -0,0 +1,36 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.util.List; + +import javax.annotation.Resource; + +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * Created by wanshao + * Date: 2017/9/20 + * Time: 下午5:00 + **/ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = {"/dal-dao.xml"}) +public class MetaHistoryDAOTest { + + @Resource + MetaHistoryDAO metaHistoryDAO; + + @Test + public void testGetAll() { + List + metaHistoryDOList = metaHistoryDAO.getAll(); + for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) { + System.out.println(metaHistoryDO.getId()); + } + } + +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java new file mode 100644 index 00000000..1eb45d7d --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java @@ -0,0 +1,47 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; + +import javax.annotation.Resource; + +import com.taobao.tddl.dbsync.binlog.BinlogPosition; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author wanshao 2017年8月2日 下午4:11:45 + * @since 3.2.5 + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(locations = { "/dal-dao.xml" }) +public class TableMetaManagerTest { + + @Resource + TableMetaManager tableMetaManager; + + @Test + public void testSimple() throws FileNotFoundException, IOException { + + + + URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); + File dummyFile = new File(url.getFile()); + File create = new File(dummyFile.getParent() + "/ddl", "create.sql"); + BinlogPosition position = BinlogPosition.parseFromString("001115:0139177334#3065927853.1501660815000"); + String createSql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); + tableMetaManager.apply(position, "tddl5_00", createSql); + + String alterSql = "alter table `test` add column name varchar(32) after c_varchar"; + position = BinlogPosition.parseFromString("001115:0139177334#3065927853.1501660816000"); + tableMetaManager.apply(position, "tddl5_00", alterSql); + + } +} diff --git a/parse/src/test/resources/binlog/tsdb/mysql-bin.000001 b/parse/src/test/resources/binlog/tsdb/mysql-bin.000001 new file mode 100644 index 0000000000000000000000000000000000000000..992efb5cf16ba259513a4c152db275e11e889d38 GIT binary patch literal 178 zcmaFAe{WJ|Uh4b(dx0pDpOJxqp&W>-ftZ1Tg~3$MT+h%_Hzz+GEI?cFOz zBakeBV!#a;G9SZ45P1m*g9$T$Oa+M{`Uk#9kak0YCKwarALw_^?d@)7yA6ZJc$0H` z?|0AlJKyhjzH{z(d#~0h%uj!(COHZr)CTQ2Xb8okyzGK(P0_+8w}TocaUw9Y3a6uZ zG!w(8TR7Utb9h#mbgqY+|uP!OB zB>?on_b~{qf}n`-(s5rA>P#$SVX=iNF|ydkx7+z<(dia!I3odLo3qY_Yn=|KAZFw& z&crQ(*Xa=WI$Tj{!4=g;BW9{Cl~zLqpp>%}7Obl*r@yM3{R6kU52>;Uh{$;xtpexb zZMc>5*c&)cM!q(46l2tBV8FMz&$o5U@ZKYmtC#n*I_-QIZ#`SeR2u_P1QGnbTOaKd&U50atgzZ@9#bB+qF+QJ}ad`KUy>I(EkNJ0;^7n0&+#N2_DrKvUq$@8vU3|EJH}3Wgoc8ZLack%CQAd}% z&ROr|Az8+8y*YTu-+fBqT4genGo~86oUOqbB-JuWx{y1#X}8j}2_yidqauy6Y$17( zWm5znia{QX2A4rQ2Jb_=(&40R89SOXhcXi~F~xW-Y}nVtt{!Jc?DrN6-4UyeCcK&A~qZy2Jjhy|;FEj`ZzQ@e49%)c>s5z<(1K4s3^{a|xKFE~GjhKg$!_ zoF5BHVl6bT$i!qqa^x=$mYpF0lvpoNN}c*I5u11kr+C&2gGzEH8e3#?GATN_Kx5(l z)pi0v$-MxEFCIKOy{WQl%mNE!yxM+yXh<1;DtY&!$l=Rj(^}y{3n8}Q&{6M6rXWLZ*K$)!sL&-Ea zA-Nk|j>l|DoVmtqVarjevD&;0!=AUoU}Vc!bHmo3rkz*DwSmG$R$PvUh7Y3}7kDla z02u+CL>PT-nUPpckRVS^rC=!Q}QtH&!1mStE) z`@@S=bhh%=X3pycnd?-TF=LgQ&CS;o<>qK~8m1_>K&Q=T7V8U3^cp>*%PlG`$}Pz$ z&}+4Mg(~4-`L-b?>kb-pWY#iUDh|-Qx9x{20zg?G14CBg7}hMghtv&Moh7Dt8=cJy z9Ue{)y|@KlBO!(cw^s}bQ)XhIqA{~ZR%gk=Cbr6G&@mQ6Wd$}HnH6lPNoO>W`Uf7W zv(?yAsgyE(Pu#bkD>=Jp$dNhAT&Xlb<@!2H8Udi3y9u`>I4NiP!E$Dxup3R(@T7WY z-x&H%Nved#6q%GvhH{7KlMbry5CBTb0fu)0AU;S6vVA@w8DS!mqS`pY+rYVz{5!VP z4%pA7J^PinTe19qoFL9%eA4Jh-SGR@1c1gl7x=}`AWz8hCuCtC9B=uDeh!2d+kptJ zBJULy^)@NCH5x^w8Cql0k~*}SCZ)Nu&ZAv82ULDe(cSP-KZ^ zY3WRWmIt2NyPW_8CBBmK4@m6o5bXc?EY8aho5gFcr1f1=N}QZZ6pAddEG?Z0=xM)v zy@vo$i8q0vhXN?Fcru%Wyp_>1E$~H7|1q#-+d?;sMQZHe1>OT6YIV4_1C#F|-h&gVW;!f&#*_u!Z+C7BPa)p-rD|NBxdiYlgk_?x%goc99Nxt=wgcX^1>)*icF!^>6I)ML8 ze4a8E*z=4wfnAMOhq+L47)M#nl9Fd|nC%Y9XfjDTHfvFS0bcadC`+!Tz!4G^89H?A z(P`D%I=n$HFc0P8HD_`aKeNizrUz9a0Ix$3Np5!?qBM0Fn0S8jn71_TkV7uGuLJYF zfl{&cHRoPH0II_T$UJR>9gjny{9^2Z-8EB;*Ld0Xa^;#r-G$i%*iV8KuR6A^$s z9{`zoprI2q?B6eI`nc*rtluQ3^h-^9cF>Uugt{AfdjyVBD(pDi;aF?Pl$&UKmf+hU zuz^nODKr~bJ8>_Xw?p%X;rPMHToin3&vt}tzdW?*UWCluq@m%cldS3EilbXH%C67e zAF@+h2S4VV;$xkQ$i)7NAiUq+#}I(G^AOPRuAQGgcWEO+5x>0ve=ZgF1C`{qo&B(# zv7y^N(k}LXw5Nvd309vw*^7c-?O7zWAvAP!wA|E_2wV_G$8S4{w!Eo-B-;*3W6oUm z$c4{A%$qu#>hHd(89fjw<@DV8NeIB3dOc~}?xv2`1f%0iB5f1*YwB>8Imvu042(h!Q8Itj2RQDG;j9JfvFPfd+&omdt=yfXq&TR#eB zdz=77!M}FLgOKf)Z!0uMX$SQ~N{lU5u8lTtJB>)ZjOt>q8K8*5;8|k=AEU zWNbB7WJ#Gj79aqzhSP<{B)Robrp|OBy77034R&iiAIMy)?=|L#Ox#3~@7gk?F9PuP zUIZH6wfCl)AqIrvkCDGoVJE31x3NWijGSDu^%~k!b0>pUYjTp=<74D_2xQBl#Rhjr z30e?XXwj$dyodGQxMgUIMZqV?K ztnEL)HWHyYS-w;VtE`*3l}gspZyueF_MEJ~PWD7rD7fvOtl+TAvk`%lWhR8z6|!1I zBn6JH(2%u;u~3I4<)g^N&6JGBEe|^ofRptLXn04~sqsP82*t_LQz4#{ReGg8+7ns5 zz`t=$_C!_~WH)d)94`CS=A$RaZ&uf+RUhWa8$EO7l$tGZBDO)ej`zQPsz1aRfqfs-{sP9#i#2 z-@itqJ*Vm~PWD7q`0r3<`=IZ1wC7Y+k^S2WRll5R{tb#bRl#?;r3nQhS;Gxe`*zE) z3kbleDg=plRGBJrS|SvuY7-UWF;xfrC&!~br|O)OJyF#KTzA*23L0&mg7%!MyI_$< z5Z+a&TCr~P6%=!-(n*XeE|bV8Vrn5>zIYY^I91C*;vH2#UE8@Ip*U4Xs1T2-+LsX3 z7wtJ!ZSTf9Fngk^D`dE%D&%;tVQ9~(>Phx%6{9qXMC%n>L7_-(uA2Jqll?hPd}WB0Gz6J z_j*g!!I?p=5Q_JzkpOiQRDB<_rv&YJuPSk}C#v)S=#Hvgo2OQxJ?~W?ll^Ihs?wf& z)}omAs_P_17hNWiQN+~V|77Q{2*9aIz0X^!q|mQMA{3`;20+~eRZG9WJ`C+SRr{Rm ziK-|7bVt>|_K8=}o>Nsr_CG3At?zf_wS^~ks8ofvLiZFY-a03dQA8*pOFn;L00IzI zBOzlN0H^~H7X3QN2tKaY`5boBD1lBng=V9}ESZe3ly0<}CCrTfa_gcsTKD$ez&+*` zsX@Ujll8d>K<+Jtb+4Ru=+1CFLAnX|dZ(u(Wu!1{BtVj~Ev9VE>XcMeV7KHJm`zgJ;0!4(bKpQJDKlfRAvFy^2c@KCNWBLS zqHFM}&E==(uL}-GklcKSK_(>|Sur~-`DPV?n^aC;T3NoeYSBlh7k!`_=G#n`(H1lK z@jIrcwy&$2zeQ_mHy4h#ap(MQFHN_|rLCS?v({O6@tEt^&O z%F4>Qv#Lus$fY}s_A##PdEL3~sq*qVwy@&_yGn-97OST8xEObZ%j#7Nr=2SQ2wafM zS1@d&N5dwJQ!}`(fvjISc}{g{`RQqMoIvu8cG%o8U~@7qZsbkPD^secY_XV}aIop@ z4rBgU&;8jtzj}UIb@5cWb*iSnF9R0Y%?5i;o;lyBx}=Yca!XOwjMAEwuiD310jF~P zmP&vvo+<;#fHwo?5`YGQXU=eYvQf$(XU&$Zwt`%T`Duqb5M;m6*Kt-^(J^7Hrc*@4 z#<}YiU{SrXys~7bYzMG~K$bK7={J{Gu3Cun&Um|WU4j!G(fB)AO{d+yzyxT4Uf*<3|VuBK|ovAXG|8gRQHSgw%opTc2be7<%_lN*|Vz zkx=9q9nTsvA-HB`S#|mARV7m^=Pa&#bz#-w+12GQLuT-WGk+@iP8-`-QD}Dm=wUIv zwH&*JUux%lg#Z-Wiy@RJc0>urb_BU~vE5!^9Ba?BK`^-!-0uYU+Zfz&?&B&L@-bAk zU_;d`WqNQYt|?E{KRB+{tlLoe@d9X~%I)t~zXVU)vf%g9T3-90;ZAi{{X9h`_o0o2 zH08Yk!EaEI4gn~%t%hLMsS}*MnYQ*xA3WrSlh*M1^I0|KMTAMY<^ppe%s@@js0k82 zqu_;?u!xPJSDxw7NK;EQIqANZtafd%E|7|1J0~Fk)$#wdVL?oBQKKPKmziHADh zy3u^}&g}Tk>=?dq@-JO~%*?fOr_Pl4i7gT@AYDa8XMHuqQ;Ti6CPYl62rUIqZC1=t{2)5p~Ute^j|tYH}`gJ=bo-bg*Tpz|@rqXp}1FntWS z!h{+;;oIM&*~;Der_fZ_qD7z%{sIH92402z(jD+8G{HY#p@Dd@{aM`?mk^&8wBrGD zo)f+(G_9?@Sj!^hlI5kn@1fJ0>D@BWjxS_yB)dcOmX{!Go0v1Y6vRPjnFPl<62pz_ zBr^VJ(E)TOmLx_a0Ov6o_O<>E9-qjH_CY-2F%t?{g!^Q|X*_WrTi4@}#bKe5LgZC^ zj}1xm6)*aTrVQ=#c_8V18hM#2Uhe7Ccy6qU`12D z3DKnVuq-S3z?O_I#t&N`~w%M6-k%soF6yGt4)eEJgWb$!|ltyIXzbD$JY= z;S{3RyR!i))0TfE;n6ZV{+fcBnanvBs2}Wh$TuPS$na=)#VI=67QWfa+a!D`GdL1b z7y8!_iaRM>!6?|LsB0<(4xvVfxJn?0BOgFPvE(d765w# z><+oNV9L0klP$@+I+5-OLNPN?_C3@uoO+E}FuZPVwP`A+XfxwcR@*3OOE|0D3qLVA zF3Q(kn#VjQmphZJrQWMpY96z|odmg5cBg8y9MZksk zi@Z%hi}+&N@_iW6c!;l{kd~^vjd~^daa*&^nh@V2PKj==FCq)=k|O`JZbShBP>3G@ znNI+K`h0{J%RT)J@IJB_MHI9 zhH^MCv6onb-nq000XSFJ2q31JtoB~)l^KB`4q|hO#BQ>{%~_NbK?mhJD`D&b1mIlF z008f}y0_T+7D92Z_K`0hb2YqmPhYg>Tm|5qi@()ET*W|!TduC2e=`>WI9H(r5T|g} JXLk09e*y1X&v5_% literal 0 HcmV?d00001 diff --git a/parse/src/test/resources/dal-dao.xml b/parse/src/test/resources/dal-dao.xml new file mode 100644 index 00000000..7d2fd0a4 --- /dev/null +++ b/parse/src/test/resources/dal-dao.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/parse/src/test/resources/ddl/create.sql b/parse/src/test/resources/ddl/create.sql new file mode 100644 index 00000000..273be888 --- /dev/null +++ b/parse/src/test/resources/ddl/create.sql @@ -0,0 +1,23 @@ + CREATE TABLE `test` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `c_tinyint` tinyint(4) DEFAULT '1' COMMENT 'tinyint', + `c_smallint` smallint(6) DEFAULT 0 COMMENT 'smallint', + `c_mediumint` mediumint(9) DEFAULT NULL COMMENT 'mediumint', + `c_int` int(11) DEFAULT NULL COMMENT 'int', + `c_bigint` bigint(20) DEFAULT NULL COMMENT 'bigint', + `c_decimal` decimal(10,3) DEFAULT NULL COMMENT 'decimal', + `c_date` date DEFAULT '0000-00-00' COMMENT 'date', + `c_datetime` datetime DEFAULT '0000-00-00 00:00:00' COMMENT 'datetime', + `c_timestamp` timestamp NULL DEFAULT NULL COMMENT 'timestamp', + `c_time` time DEFAULT NULL COMMENT 'time', + `c_char` char(10) DEFAULT NULL COMMENT 'char', + `c_varchar` varchar(10) DEFAULT 'hello' COMMENT 'varchar', + `c_blob` blob COMMENT 'blob', + `c_text` text COMMENT 'text', + `c_mediumtext` mediumtext COMMENT 'mediumtext', + `c_longblob` longblob COMMENT 'longblob', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_a` (`c_tinyint`), + KEY `k_b` (`c_smallint`), + KEY `k_c` (`c_mediumint`,`c_int`) +) ENGINE=InnoDB AUTO_INCREMENT=1769503 DEFAULT CHARSET=utf8mb4 COMMENT='10000000'; \ No newline at end of file diff --git a/pom.xml b/pom.xml index 29a08fcb..cb5f89cd 100644 --- a/pom.xml +++ b/pom.xml @@ -135,6 +135,12 @@ spring 2.5.6 + + org.springframework + spring-test + 2.5.6 + test + commons-lang @@ -204,6 +210,16 @@ protobuf-java 2.6.1 + + org.apache.ibatis + ibatis-sqlmap + 2.3.4.726 + + + com.alibaba + druid + 1.1.3 + ch.qos.logback @@ -236,7 +252,7 @@ mysql mysql-connector-java 5.1.33 - test + diff --git a/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java b/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java index 10f2f3e5..afc74c86 100644 --- a/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java +++ b/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java @@ -19,12 +19,12 @@ import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; public abstract class BaseCanalServerWithEmbededTest { protected static final String cluster1 = "127.0.0.1:2188"; - protected static final String DESTINATION = "ljhtest1"; + protected static final String DESTINATION = "example"; protected static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()"; protected static final String MYSQL_ADDRESS = "127.0.0.1"; - protected static final String USERNAME = "retl"; - protected static final String PASSWORD = "retl"; - protected static final String FILTER = "retl\\..*,erosa.zk_complaint_bizdata"; + protected static final String USERNAME = "canal"; + protected static final String PASSWORD = "canal"; + protected static final String FILTER = ".*\\\\..*"; private CanalServerWithEmbedded server; private ClientIdentity clientIdentity = new ClientIdentity(DESTINATION, (short) 1); ; diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_StandaloneTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_StandaloneTest.java index 9790f770..a4ce077a 100644 --- a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_StandaloneTest.java +++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbedded_StandaloneTest.java @@ -34,8 +34,8 @@ public class CanalServerWithEmbedded_StandaloneTest extends BaseCanalServerWithE new InetSocketAddress(MYSQL_ADDRESS, 3306))); parameter.setDbUsername(USERNAME); parameter.setDbPassword(PASSWORD); - parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}", - "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}")); + parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000003\",\"position\":14217L,\"timestamp\":\"1505998863000\"}", + "{\"journalName\":\"mysql-bin.000003\",\"position\":14377L,\"timestamp\":\"1505998863000\"}")); parameter.setSlaveId(1234L); From 19707f92d4e756585488b2fc7613ff424f7f0baa Mon Sep 17 00:00:00 2001 From: agapple Date: Sat, 30 Sep 2017 23:21:17 +0800 Subject: [PATCH 02/34] merge tsdb meta support and refactor --- .../tddl/dbsync/binlog/BinlogPosition.java | 12 +- .../resources/example/instance.properties | 9 +- .../main/resources/spring/file-instance.xml | 6 +- .../resources/spring/{ => tsdb}/dal-dao.xml | 5 +- .../spring/{ => tsdb}/mybatis-config.xml | 0 .../spring/tsdb}/sql-map/sqlmap_history.xml | 6 - .../{ => tsdb}/sql-map/sqlmap_snapshot.xml | 0 .../spring/{ => tsdb}/sqlmap-config.xml | 0 .../parse/driver/mysql/MysqlConnector.java | 2 +- .../driver/mysql/MysqlQueryExecutor.java | 57 ++- .../client/ClientAuthenticationPacket.java | 7 +- .../parse/inbound/AbstractEventParser.java | 6 +- .../canal/parse/inbound/BinlogParser.java | 2 +- .../otter/canal/parse/inbound/TableMeta.java | 24 +- .../mysql/AbstractMysqlEventParser.java | 8 +- .../inbound/mysql/LocalBinlogEventParser.java | 2 +- .../parse/inbound/mysql/MysqlConnection.java | 74 +--- .../parse/inbound/mysql/MysqlEventParser.java | 20 +- .../inbound/mysql/dbsync/DBMSAction.java | 77 ---- .../inbound/mysql/dbsync/LogEventConvert.java | 210 ++++++---- .../inbound/mysql/dbsync/TableMetaCache.java | 165 ++++---- .../parse/inbound/mysql/ddl/DdlResult.java | 39 +- .../inbound/mysql/ddl/DruidDdlParser.java | 34 +- .../{dbsync => ddl}/SimpleDdlParser.java | 100 +---- .../mysql/tsdb/DataSourceFactoryTSDB.java | 52 ++- .../inbound/mysql/tsdb/MemoryTableMeta.java | 6 +- .../inbound/mysql/tsdb/TableMetaManager.java | 390 +++++++----------- .../mysql/tsdb/dao/MetaHistoryDAO.java | 11 +- .../mysql/tsdb/dao/MetaSnapshotDAO.java | 10 +- .../mysql/tsdb/model/MetaHistoryDO.java | 10 +- .../mysql/tsdb/model/MetaSnapshotDO.java | 10 +- .../index/FileMixedLogPositionManager.java | 1 + .../index/PeriodMixedLogPositionManager.java | 1 + .../parse/inbound/TableMetaCacheTest.java | 53 ++- .../inbound/group/GroupEventPaserTest.java | 1 - .../inbound/mysql/SimpleDdlParserTest.java | 4 +- .../mysql/tsdb/MemoryTableMetaTest.java | 4 +- .../mysql/tsdb/MetaHistoryDAOTest.java | 9 +- .../mysql/tsdb/TableMetaManagerTest.java | 2 - parse/src/test/resources/dal-dao.xml | 46 --- .../resources/tsdb}/dal-dao.xml | 9 +- .../resources/tsdb}/mybatis-config.xml | 0 .../tsdb}/sql-map/sqlmap_history.xml | 0 .../tsdb}/sql-map/sqlmap_snapshot.xml | 0 .../resources/tsdb}/sqlmap-config.xml | 0 45 files changed, 585 insertions(+), 899 deletions(-) rename deployer/src/main/resources/spring/{ => tsdb}/dal-dao.xml (89%) rename deployer/src/main/resources/spring/{ => tsdb}/mybatis-config.xml (100%) rename {parse/src/main/resources => deployer/src/main/resources/spring/tsdb}/sql-map/sqlmap_history.xml (99%) rename deployer/src/main/resources/spring/{ => tsdb}/sql-map/sqlmap_snapshot.xml (100%) rename deployer/src/main/resources/spring/{ => tsdb}/sqlmap-config.xml (100%) delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java rename parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/{dbsync => ddl}/SimpleDdlParser.java (73%) delete mode 100644 parse/src/test/resources/dal-dao.xml rename parse/src/{main/resources => test/resources/tsdb}/dal-dao.xml (83%) rename parse/src/{main/resources => test/resources/tsdb}/mybatis-config.xml (100%) rename {deployer/src/main/resources/spring => parse/src/test/resources/tsdb}/sql-map/sqlmap_history.xml (100%) rename parse/src/{main/resources => test/resources/tsdb}/sql-map/sqlmap_snapshot.xml (100%) rename parse/src/{main/resources => test/resources/tsdb}/sqlmap-config.xml (100%) diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java index 92bba817..a602c05f 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java @@ -1,6 +1,5 @@ package com.taobao.tddl.dbsync.binlog; -import com.taobao.tddl.dbsync.binlog.LogPosition; /** * Position inside binlog file @@ -35,9 +34,8 @@ public class BinlogPosition extends LogPosition { } private final static long[] pow10 = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, - 10000000000L, 100000000000L, 1000000000000L, 10000000000000L, - 100000000000000L, 1000000000000000L, 10000000000000000L, 100000000000000000L, - 1000000000000000000L }; + 10000000000L, 100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, + 10000000000000000L, 100000000000000000L, 1000000000000000000L }; public static String placeHolder(int bit, long number) { if (bit > 18) { @@ -53,9 +51,6 @@ public class BinlogPosition extends LogPosition { return String.valueOf(max + number).substring(1); } - /** - * Return BinlogPosition in String representation. This serves as EventId for DBMSEvent. - */ public String format2String(final int positionMaxLen) { String binlogSuffix = fileName; String binlogOffset = placeHolder((int) positionMaxLen, position); @@ -90,7 +85,8 @@ public class BinlogPosition extends LogPosition { if (sharpIndex != -1) { binlogPosition = Long.parseLong(source.substring(miscIndex, sharpIndex)); } else if (semicolonIndex != -1) { - binlogPosition = Long.parseLong(source.substring(miscIndex, semicolonIndex)); // NOTE: 向后兼容 + binlogPosition = Long.parseLong(source.substring(miscIndex, semicolonIndex)); // NOTE: + // 向后兼容 } else if (dotIndex != -1) { binlogPosition = Long.parseLong(source.substring(miscIndex, dotIndex)); } else { diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 3c5e0a8e..7f00f079 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -9,15 +9,10 @@ canal.instance.master.timestamp=1506088042 # tsdb info -canal.instance.tsdb.address=127.0.0.1:3306 canal.instance.tsdb.enable=true +canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/tsdb canal.instance.tsdb.dbUsername=canal canal.instance.tsdb.dbPassword=canal -# tsdb关联的目标db，和canal.instance.defaultDatabaseName意义相同 -canal.instance.tsdb.defaultDatabaseName=test -canal.instance.tsdb.connectionCharset=UTF-8 -canal.instance.tsdb.driverClassName=com.mysql.jdbc.Driver -canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/tsdb #canal.instance.standby.address = @@ -31,6 +26,6 @@ canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 # table regex canal.instance.filter.regex=.*\\..* -# table black regex , 默认过滤表结构管理的数据 +# table black regex canal.instance.filter.black.regex=tsdb\\..* ################################################# \ No newline at end of file diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index a2563119..a9f2823b 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -180,13 +180,10 @@ - + - - - @@ -210,5 +207,4 @@ - diff --git a/deployer/src/main/resources/spring/dal-dao.xml b/deployer/src/main/resources/spring/tsdb/dal-dao.xml similarity index 89% rename from deployer/src/main/resources/spring/dal-dao.xml rename to deployer/src/main/resources/spring/tsdb/dal-dao.xml index 644979fc..1842641f 100644 --- a/deployer/src/main/resources/spring/dal-dao.xml +++ b/deployer/src/main/resources/spring/tsdb/dal-dao.xml @@ -9,13 +9,10 @@ - + - - - diff --git a/deployer/src/main/resources/spring/mybatis-config.xml b/deployer/src/main/resources/spring/tsdb/mybatis-config.xml similarity index 100% rename from deployer/src/main/resources/spring/mybatis-config.xml rename to deployer/src/main/resources/spring/tsdb/mybatis-config.xml diff --git a/parse/src/main/resources/sql-map/sqlmap_history.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml similarity index 99% rename from parse/src/main/resources/sql-map/sqlmap_history.xml rename to deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml index 00197194..a983e222 100644 --- a/parse/src/main/resources/sql-map/sqlmap_history.xml +++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml @@ -6,18 +6,14 @@ @@ -42,10 +38,8 @@ diff --git a/deployer/src/main/resources/spring/sql-map/sqlmap_snapshot.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml similarity index 100% rename from deployer/src/main/resources/spring/sql-map/sqlmap_snapshot.xml rename to deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml diff --git a/deployer/src/main/resources/spring/sqlmap-config.xml b/deployer/src/main/resources/spring/tsdb/sqlmap-config.xml similarity index 100% rename from deployer/src/main/resources/spring/sqlmap-config.xml rename to deployer/src/main/resources/spring/tsdb/sqlmap-config.xml diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 49f6edea..0e488162 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -34,7 +34,7 @@ public class MysqlConnector { private byte charsetNumber = 33; private String defaultSchema = "retl"; private int soTimeout = 30 * 1000; - private int connTimeout = 5 * 1000; + private int connTimeout = 5 * 1000; private int receiveBufferSize = 16 * 1024; private int sendBufferSize = 16 * 1024; diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java index a389623e..6d30bd4f 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java @@ -6,6 +6,7 @@ import java.util.List; import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.EOFPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetHeaderPacket; @@ -93,11 +94,65 @@ public class MysqlQueryExecutor { return resultSet; } - private void readEofPacket() throws IOException { + public List queryMulti(String queryString) throws IOException { + QueryCommandPacket cmd = new QueryCommandPacket(); + cmd.setQueryString(queryString); + byte[] bodyBytes = cmd.toBytes(); + PacketManager.writeBody(channel, bodyBytes); + List resultSets = new ArrayList(); + boolean moreResult = true; + while (moreResult) { + byte[] body = readNextPacket(); + if (body[0] < 0) { + ErrorPacket packet = new ErrorPacket(); + packet.fromBytes(body); + throw new IOException(packet + "\n with command: " + queryString); + } + + ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket(); + rsHeader.fromBytes(body); + + List fields = new ArrayList(); + for (int i = 0; i < rsHeader.getColumnCount(); i++) { + FieldPacket fp = new FieldPacket(); + fp.fromBytes(readNextPacket()); + fields.add(fp); + } + + moreResult = readEofPacket(); + + List rowData = new ArrayList(); + while (true) { + body = readNextPacket(); + if (body[0] == -2) { + break; + } + RowDataPacket rowDataPacket = new RowDataPacket(); + rowDataPacket.fromBytes(body); + rowData.add(rowDataPacket); + } + + ResultSetPacket resultSet = new ResultSetPacket(); + resultSet.getFieldDescriptors().addAll(fields); + for (RowDataPacket r : rowData) { + resultSet.getFieldValues().addAll(r.getColumns()); + } + resultSet.setSourceAddress(channel.getRemoteSocketAddress()); + resultSets.add(resultSet); + } + + return resultSets; + } + + private boolean readEofPacket() throws IOException { byte[] eofBody = readNextPacket(); + EOFPacket packet = new EOFPacket(); + packet.fromBytes(eofBody); if (eofBody[0] != -2) { throw new IOException("EOF Packet is expected, but packet with field_count=" + eofBody[0] + " is found."); } + + return (packet.statusFlag & 0x0008) != 0; } protected byte[] readNextPacket() throws IOException { diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java index 04975a31..fb04d7dc 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java @@ -47,10 +47,11 @@ public class ClientAuthenticationPacket extends PacketWithHeaderPacket { /** * CLIENT_LONG_PASSWORD CLIENT_LONG_FLAG CLIENT_PROTOCOL_41 * CLIENT_INTERACTIVE CLIENT_TRANSACTIONS CLIENT_SECURE_CONNECTION + * CLIENT_MULTI_STATEMENTS; */ - ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768, out); // remove - // client_interactive - // feature + ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768 | 0x00010000, out); // remove + // client_interactive + // feature // 2. write max_packet_size ByteHelper.writeUnsignedIntLittleEndian(MSC.MAX_PACKET_LENGTH, out); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index b5e1b090..32d3aee5 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -173,7 +173,7 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle public boolean sink(EVENT event) { try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,false); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, false); if (!running) { return false; @@ -320,13 +320,13 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle return result; } - protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod,boolean isSeek) throws Exception { + protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception { long startTs = -1; boolean enabled = getProfilingEnabled(); if (enabled) { startTs = System.currentTimeMillis(); } - CanalEntry.Entry event = binlogParser.parse(bod,isSeek); + CanalEntry.Entry event = binlogParser.parse(bod, isSeek); if (enabled) { this.parsingInterval = System.currentTimeMillis() - startTs; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java index 01c55a51..034879e0 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java @@ -11,7 +11,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry; */ public interface BinlogParser extends CanalLifeCycle { - CanalEntry.Entry parse(T event,boolean isSeek) throws CanalParseException; + CanalEntry.Entry parse(T event, boolean isSeek) throws CanalParseException; void reset(); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java index d3009624..7f56e440 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java @@ -20,16 +20,16 @@ import org.apache.commons.lang.StringUtils; */ public class TableMeta { - private String schema; - private String table; + private String schema; + private String table; private List fields = new ArrayList(); - private String ddl; // 表结构的DDL语句 + private String ddl; // 表结构的DDL语句 - public TableMeta() { + public TableMeta(){ } - public TableMeta(String schema, String table, List fields) { + public TableMeta(String schema, String table, List fields){ this.schema = schema; this.table = table; this.fields = fields; @@ -109,11 +109,11 @@ public class TableMeta { public static class FieldMeta { - public FieldMeta() { + public FieldMeta(){ } - public FieldMeta(String columnName, String columnType, boolean nullable, boolean key, String defaultValue) { + public FieldMeta(String columnName, String columnType, boolean nullable, boolean key, String defaultValue){ this.columnName = columnName; this.columnType = columnType; this.nullable = nullable; @@ -121,12 +121,12 @@ public class TableMeta { this.defaultValue = defaultValue; } - private String columnName; - private String columnType; + private String columnName; + private String columnType; private boolean nullable; private boolean key; - private String defaultValue; - private String extra; + private String defaultValue; + private String extra; public String getColumnName() { return columnName; @@ -182,7 +182,7 @@ public class TableMeta { public String toString() { return "FieldMeta [columnName=" + columnName + ", columnType=" + columnType + ", defaultValue=" - + defaultValue + ", nullable=" + nullable + ", key=" + key + "]"; + + defaultValue + ", nullable=" + nullable + ", key=" + key + "]"; } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 4510159b..094d0b1d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -30,8 +30,8 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { protected boolean filterTableError = false; @Resource - protected TableMetaManager tableMetaManager; - protected boolean useDruidDdlFilter = true; + protected TableMetaManager tableMetaManager; + protected boolean useDruidDdlFilter = true; protected BinlogParser buildParser() { LogEventConvert convert = new LogEventConvert(); @@ -50,7 +50,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { convert.setFilterRows(filterRows); convert.setFilterTableError(filterTableError); - //初始化parser的时候也初始化管理mysql 表结构的管理器 + // 初始化parser的时候也初始化管理mysql 表结构的管理器 tableMetaManager.init(); return convert; } @@ -66,6 +66,7 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { /** * 回滚到指定位点 + * * @param position * @return */ @@ -77,7 +78,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { return true; } - public void setEventBlackFilter(CanalEventFilter eventBlackFilter) { super.setEventBlackFilter(eventBlackFilter); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java index 87f31208..d45c5615 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java @@ -50,7 +50,7 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements throw new CanalParseException(e); } - tableMetaCache = new TableMetaCache(metaConnection,tableMetaManager); + tableMetaCache = new TableMetaCache(metaConnection, tableMetaManager); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index e399a356..94625b10 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -3,14 +3,8 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.Charset; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; import java.util.List; -import java.util.Properties; -import com.mysql.jdbc.Driver; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,28 +21,24 @@ import com.alibaba.otter.canal.parse.inbound.ErosaConnection; import com.alibaba.otter.canal.parse.inbound.SinkFunction; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; - import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; public class MysqlConnection implements ErosaConnection { - private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class); + private static final Logger logger = LoggerFactory.getLogger(MysqlConnection.class); private MysqlConnector connector; private long slaveId; - private Charset charset = Charset.forName("UTF-8"); + private Charset charset = Charset.forName("UTF-8"); private BinlogFormat binlogFormat; private BinlogImage binlogImage; // tsdb releated - private AuthenticationInfo authInfo; - private Connection conn; - protected int connTimeout = 5 * 1000; // 5秒 - protected int soTimeout = 60 * 60 * 1000; // 1小时 - protected int bufferSize = 16 * 1024; - + private AuthenticationInfo authInfo; + protected int connTimeout = 5 * 1000; // 5秒 + protected int soTimeout = 60 * 60 * 1000; // 1小时 public MysqlConnection(){ } @@ -59,7 +49,7 @@ public class MysqlConnection implements ErosaConnection { authInfo.setUsername(username); authInfo.setPassword(password); connector = new MysqlConnector(address, username, password); - //将connection里面的参数透传下 + // 将connection里面的参数透传下 connector.setSoTimeout(soTimeout); connector.setConnTimeout(connTimeout); } @@ -72,29 +62,13 @@ public class MysqlConnection implements ErosaConnection { authInfo.setPassword(password); authInfo.setDefaultDatabaseName(defaultSchema); connector = new MysqlConnector(address, username, password, charsetNumber, defaultSchema); - //将connection里面的参数透传下 + // 将connection里面的参数透传下 connector.setSoTimeout(soTimeout); connector.setConnTimeout(connTimeout); } public void connect() throws IOException { connector.connect(); - //准备一个connection连接给tsdb查询用 - Properties info = new Properties(); - info.put("user", authInfo.getUsername()); - info.put("password", authInfo.getPassword()); - info.put("connectTimeout", String.valueOf(connTimeout)); - info.put("socketTimeout", String.valueOf(soTimeout)); - String url = "jdbc:mysql://" + authInfo.getAddress().getHostName() + ":" - + String.valueOf(authInfo.getAddress().getPort()) + "?allowMultiQueries=true"; - try { - Driver driver = new com.mysql.jdbc.Driver(); - conn = driver.connect(url, info); - // conn = DriverManager.getConnection(url, info); - } catch (SQLException e) { - throw new CanalParseException(e); - } - } public void reconnect() throws IOException { @@ -114,23 +88,9 @@ public class MysqlConnection implements ErosaConnection { return exector.query(cmd); } - public T query(String sql, ProcessJdbcResult processor) { - Statement stmt = null; - try { - stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery(sql); - return processor.process(rs); - } catch (SQLException e) { - throw new CanalParseException(e); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - // ignore - } - } - } + public List queryMulti(String cmd) throws IOException { + MysqlQueryExecutor exector = new MysqlQueryExecutor(connector); + return exector.queryMulti(cmd); } public void update(String cmd) throws IOException { @@ -212,7 +172,7 @@ public class MysqlConnection implements ErosaConnection { connection.setCharset(getCharset()); connection.setSlaveId(getSlaveId()); connection.setConnector(connector.fork()); - //set authInfo + // set authInfo connection.setAuthInfo(authInfo); return connection; } @@ -449,9 +409,6 @@ public class MysqlConnection implements ErosaConnection { return binlogImage; } - public Connection getConn() { - return conn; - } public InetSocketAddress getAddress() { return authInfo.getAddress(); } @@ -464,10 +421,6 @@ public class MysqlConnection implements ErosaConnection { this.soTimeout = soTimeout; } - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - public AuthenticationInfo getAuthInfo() { return authInfo; } @@ -475,9 +428,4 @@ public class MysqlConnection implements ErosaConnection { public void setAuthInfo(AuthenticationInfo authInfo) { this.authInfo = authInfo; } - - public static interface ProcessJdbcResult { - - public T process(ResultSet rs) throws SQLException; - } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index e6dedbdb..4a188b01 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -68,7 +68,6 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // update by yishun.chen,特殊异常处理参数 private int dumpErrorCount = 0; // binlogDump失败异常计数 - private int dumpTimeoutCount = 0;// socketTimeout异常 private int dumpErrorCountThreshold = 2; // binlogDump失败异常计数阀值 protected ErosaConnection buildErosaConnection() { @@ -118,12 +117,12 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } } - if(tableMetaManager != null){ + if (tableMetaManager != null) { tableMetaManager.setConnection(metaConnection); tableMetaManager.setFilter(eventFilter); } - tableMetaCache = new TableMetaCache(metaConnection,tableMetaManager); + tableMetaCache = new TableMetaCache(metaConnection, tableMetaManager); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } } @@ -443,7 +442,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE public boolean sink(LogEvent event) { try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,true); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true); if (entry == null) { return true; } @@ -476,7 +475,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE public boolean sink(LogEvent event) { try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,true); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true); if (entry == null) { return true; } @@ -555,7 +554,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } } catch (Exception e) { logger.warn(String.format("the binlogfile:%s doesn't exist, to continue to search the next binlogfile , caused by", - startSearchBinlogFile), e); + startSearchBinlogFile), + e); int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1)); if (binlogSeqNum <= 1) { logger.warn("Didn't find the corresponding binlog files"); @@ -687,7 +687,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE public boolean sink(LogEvent event) { EntryPosition entryPosition = null; try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event,true); + CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true); if (entry == null) { return true; } @@ -707,7 +707,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } if (StringUtils.equals(endPosition.getJournalName(), logfilename) - && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) { + && endPosition.getPosition() <= logfileoffset) { return false; } @@ -715,9 +715,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // position = current + // data.length，代表该事务的下一条offest，避免多余的事务重复 if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) { - entryPosition = new EntryPosition(logfilename, - logfileoffset + event.getEventLen(), - logposTimestamp); + entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp); logger.debug("set {} to be pending start position before finding another proper one...", entryPosition); logPosition.setPostion(entryPosition); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java deleted file mode 100644 index 4efbdc64..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DBMSAction.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.dbsync; - -/** - * Defines database change action types: INSERT, UPDATE, DELETE, OTHER. - * - * @author Changyuan.lh - * @version 1.0 - */ -public enum DBMSAction { - - INSERT('I'), UPDATE('U'), DELETE('D'), REPLACE('R'), OTHER('O'), CREATE('C'), ALTER('A'), ERASE('E'), QUERY('Q'), - ROWQUERY('W'), TRUNCATE('T'), CINDEX('X'), DINDEX('Y'), RENAME('Z'); - - protected final byte bValue; - - DBMSAction(char ch){ - this.bValue = (byte) ch; - } - - /** - * Return action type from byte value. - */ - public static DBMSAction fromValue(int iValue) { - switch ((char) iValue) { - case 'I': - case 'M': // MERGE (Oracle only) - return INSERT; - case 'U': - return UPDATE; - case 'D': // DELETE - return DELETE; - case 'R': // REPLACE - return REPLACE; - case 'C': - return CREATE; - case 'A': - return ALTER; - case 'E': - return ERASE; - case 'Q': - return QUERY; - case 'W': - return ROWQUERY; - case 'T': - return TRUNCATE; - case 'X': - return CINDEX; - case 'Y': - return DINDEX; - case 'Z': - return RENAME; - } - return OTHER; - } - - /** - * Return action type from query. - */ - public static DBMSAction fromQuery(String query) { - int length = query.length(); - for (int index = 0; index < length; index++) { - char ch = query.charAt(index); - if (!Character.isWhitespace(ch)) { - return DBMSAction.fromValue(ch); - } - } - - return OTHER; - } - - /** - * Return byte value of action type. - */ - public byte value() { - return bValue; - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 222d3010..4f0ada45 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -9,7 +9,6 @@ import java.sql.Types; import java.util.BitSet; import java.util.List; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.slf4j.Logger; @@ -22,7 +21,9 @@ import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException; import com.alibaba.otter.canal.parse.inbound.BinlogParser; import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.SimpleDdlParser; import com.alibaba.otter.canal.protocol.CanalEntry.Column; import com.alibaba.otter.canal.protocol.CanalEntry.Entry; import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; @@ -35,6 +36,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin; import com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd; import com.alibaba.otter.canal.protocol.CanalEntry.Type; import com.google.protobuf.ByteString; +import com.taobao.tddl.dbsync.binlog.BinlogPosition; import com.taobao.tddl.dbsync.binlog.LogEvent; import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent; @@ -87,6 +89,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar private boolean filterTableError = false; // 新增rows过滤，用于仅订阅除rows以外的数据 private boolean filterRows = false; + private boolean useDruidDdlFilter = true; @Override public Entry parse(LogEvent logEvent, boolean isSeek) throws CanalParseException { @@ -100,7 +103,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar binlogFileName = ((RotateLogEvent) logEvent).getFilename(); break; case LogEvent.QUERY_EVENT: - return parseQueryEvent((QueryLogEvent) logEvent,isSeek); + return parseQueryEvent((QueryLogEvent) logEvent, isSeek); case LogEvent.XID_EVENT: return parseXidEvent((XidLogEvent) logEvent); case LogEvent.TABLE_MAP_EVENT: @@ -150,110 +153,139 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar Header header = createHeader(binlogFileName, event.getHeader(), "", "", null); return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString()); } else { - // DDL语句处理 - DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName()); - - String schemaName = event.getDbName(); - if (StringUtils.isNotEmpty(result.getSchemaName())) { - schemaName = result.getSchemaName(); - } - - String tableName = result.getTableName(); + boolean notFilter = false; EventType type = EventType.QUERY; - // fixed issue https://github.com/alibaba/canal/issues/58 - if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE - || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE - || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX - || result.getType() == EventType.DINDEX) { // 针对DDL类型 - - if (filterQueryDdl) { - return null; + String tableName = null; + String schemaName = null; + if (useDruidDdlFilter) { + List results = DruidDdlParser.parse(queryString, event.getDbName()); + for (DdlResult result : results) { + if (!processFilter(queryString, result)) { + // 只要有一个数据不进行过滤 + notFilter = true; + } + } + if (results.size() > 0) { + // 如果针对多行的DDL,只能取第一条 + type = results.get(0).getType(); + schemaName = results.get(0).getSchemaName(); + tableName = results.get(0).getTableName(); + } + } else { + DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName()); + if (!processFilter(queryString, result)) { + notFilter = true; } type = result.getType(); - if (StringUtils.isEmpty(tableName) - || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) { - // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析 - throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " - + queryString + " , and DdlResult: " + result.toString()); - // return null; - } else { - // check name filter - String name = schemaName + "." + tableName; - if (nameFilter != null && !nameFilter.filter(name)) { - if (result.getType() == EventType.RENAME) { - // rename校验只要源和目标满足一个就进行操作 - if (nameFilter != null - && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) { - return null; - } - } else { - // 其他情况返回null - return null; - } - } + schemaName = result.getSchemaName(); + tableName = result.getTableName(); + } - if (nameBlackFilter != null && nameBlackFilter.filter(name)) { - if (result.getType() == EventType.RENAME) { - // rename校验只要源和目标满足一个就进行操作 - if (nameBlackFilter != null - && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) { - return null; - } - } else { - // 其他情况返回null - return null; - } - } - } - } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE - || result.getType() == EventType.DELETE) { - // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤 - if (filterQueryDml) { - return null; - } - } else if (filterQueryDcl) { + if (!notFilter) { + // 如果是过滤的数据就不处理了 return null; } - // 更新下table meta cache - //if (tableMetaCache != null - // && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) { - // for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) { - // String schemaName0 = event.getDbName(); // 防止rename语句后产生schema变更带来影响 - // if (StringUtils.isNotEmpty(renameResult.getSchemaName())) { - // schemaName0 = renameResult.getSchemaName(); - // } - // - // tableName = renameResult.getTableName(); - // if (StringUtils.isNotEmpty(tableName)) { - // // 如果解析到了正确的表信息，则根据全名进行清除 - // tableMetaCache.clearTableMeta(schemaName0, tableName); - // } else { - // // 如果无法解析正确的表信息，则根据schema进行清除 - // tableMetaCache.clearTableMetaWithSchemaName(schemaName0); - // } - // } - //} - - //使用新的表结构元数据管理方式 - BinlogPosition position = createPosition(event.getHeader()); - tableMetaCache.apply(position, event.getDbName(), queryString); + if (!isSeek) { + // 使用新的表结构元数据管理方式 + BinlogPosition position = createPosition(event.getHeader()); + tableMetaCache.apply(position, event.getDbName(), queryString); + } Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); - if (result.getType() != EventType.QUERY) { + if (type != EventType.QUERY) { rowChangeBuider.setIsDdl(true); } rowChangeBuider.setSql(queryString); if (StringUtils.isNotEmpty(event.getDbName())) {// 可能为空 rowChangeBuider.setDdlSchemaName(event.getDbName()); } - rowChangeBuider.setEventType(result.getType()); + rowChangeBuider.setEventType(type); return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString()); } } + private boolean processFilter(String queryString, DdlResult result) { + String schemaName = result.getSchemaName(); + String tableName = result.getTableName(); + // fixed issue https://github.com/alibaba/canal/issues/58 + // 更新下table meta cache + if (tableMetaCache != null + && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) { + // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤 + for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) { + String schemaName0 = renameResult.getSchemaName(); + String tableName0 = renameResult.getTableName(); + if (StringUtils.isNotEmpty(tableName0)) { + // 如果解析到了正确的表信息，则根据全名进行清除 + tableMetaCache.clearTableMeta(schemaName0, tableName0); + } else { + // 如果无法解析正确的表信息，则根据schema进行清除 + tableMetaCache.clearTableMetaWithSchemaName(schemaName0); + } + } + } + + // fixed issue https://github.com/alibaba/canal/issues/58 + if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE + || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE + || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX + || result.getType() == EventType.DINDEX) { // 针对DDL类型 + + if (filterQueryDdl) { + return true; + } + + if (StringUtils.isEmpty(tableName) + || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) { + // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析 + throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " + + queryString + " , and DdlResult: " + result.toString()); + // return null; + } else { + // check name filter + String name = schemaName + "." + tableName; + if (nameFilter != null && !nameFilter.filter(name)) { + if (result.getType() == EventType.RENAME) { + // rename校验只要源和目标满足一个就进行操作 + if (nameFilter != null + && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) { + return true; + } + } else { + // 其他情况返回null + return true; + } + } + + if (nameBlackFilter != null && nameBlackFilter.filter(name)) { + if (result.getType() == EventType.RENAME) { + // rename校验只要源和目标满足一个就进行操作 + if (nameBlackFilter != null + && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) { + return true; + } + } else { + // 其他情况返回null + return true; + } + } + } + } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE + || result.getType() == EventType.DELETE) { + // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤 + if (filterQueryDml) { + return true; + } + } else if (filterQueryDcl) { + return true; + } + + return false; + } + private Entry parseRowsQueryEvent(RowsQueryLogEvent event) { if (filterQueryDml) { return null; @@ -366,7 +398,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar boolean tableError = false; TableMeta tableMeta = null; if (tableMetaCache != null) {// 入错存在table meta cache - tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true,position); + tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position); if (tableMeta == null) { tableError = true; if (!filterTableError) { @@ -444,7 +476,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作 // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作 // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作 - tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false,position);// 强制重新获取一次 + tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false, position);// 强制重新获取一次 if (tableMeta == null) { tableError = true; if (!filterTableError) { @@ -686,7 +718,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, BinlogPosition position) { try { - return tableMetaCache.getTableMeta(dbName, tbName, useCache,position); + return tableMetaCache.getTableMeta(dbName, tbName, useCache, position); } catch (Exception e) { String message = ExceptionUtils.getRootCauseMessage(e); if (filterTableError) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index ff0e720f..23229859 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -1,24 +1,24 @@ package com.alibaba.otter.canal.parse.inbound.mysql.dbsync; import java.io.IOException; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.lang.StringUtils; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.ProcessJdbcResult; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.taobao.tddl.dbsync.binlog.BinlogPosition; /** * 处理table meta解析和缓存 @@ -28,57 +28,23 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; */ public class TableMetaCache { - public static final String COLUMN_NAME = "COLUMN_NAME"; - public static final String COLUMN_TYPE = "COLUMN_TYPE"; - public static final String IS_NULLABLE = "IS_NULLABLE"; - public static final String COLUMN_KEY = "COLUMN_KEY"; - public static final String COLUMN_DEFAULT = "COLUMN_DEFAULT"; - public static final String EXTRA = "EXTRA"; - private MysqlConnection connection; - private boolean isOnRDS = false; + public static final String COLUMN_NAME = "COLUMN_NAME"; + public static final String COLUMN_TYPE = "COLUMN_TYPE"; + public static final String IS_NULLABLE = "IS_NULLABLE"; + public static final String COLUMN_KEY = "COLUMN_KEY"; + public static final String COLUMN_DEFAULT = "COLUMN_DEFAULT"; + public static final String EXTRA = "EXTRA"; + private MysqlConnection connection; + private boolean isOnRDS = false; - private TableMetaManager tableMetaManager; + private TableMetaManager tableMetaManager; // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表 private LoadingCache tableMetaDB; - - // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表 - private Map tableMetaCache; - - /** - * 从db获取表结构 - * @param fullname - * @return - */ - private TableMeta getTableMetaByDB(final String fullname) { - return connection.query("desc " + fullname, new ProcessJdbcResult() { - - @Override - public TableMeta process(ResultSet rs) throws SQLException { - List metas = new ArrayList(); - while (rs.next()) { - FieldMeta meta = new FieldMeta(); - // 做一个优化，使用String.intern()，共享String对象，减少内存使用 - meta.setColumnName(rs.getString("Field")); - meta.setColumnType(rs.getString("Type")); - meta.setNullable(StringUtils.equalsIgnoreCase(rs.getString("Null"), "YES")); - meta.setKey("PRI".equalsIgnoreCase(rs.getString("Key"))); - meta.setDefaultValue(rs.getString("Default")); - metas.add(meta); - } - - String[] names = StringUtils.split(fullname, "`.`"); - String schema = names[0]; - String table = names[1].substring(0, names[1].length()); - return new TableMeta(schema, table, metas); - } - }); - } - - public TableMetaCache(MysqlConnection con,TableMetaManager tableMetaManager) { + public TableMetaCache(MysqlConnection con, TableMetaManager tableMetaManager){ this.connection = con; this.tableMetaManager = tableMetaManager; - //如果持久存储的表结构为空，从db里面获取下 + // 如果持久存储的表结构为空，从db里面获取下 if (tableMetaManager == null) { this.tableMetaDB = CacheBuilder.newBuilder().build(new CacheLoader() { @@ -100,16 +66,62 @@ public class TableMetaCache { }); } - isOnRDS = connection.query("show global variables like 'rds\\_%'", new ProcessJdbcResult() { - - @Override - public Boolean process(ResultSet rs) throws SQLException { - if (rs.next()) { - return true; - } - return false; + try { + ResultSetPacket packet = connection.query("show global variables like 'rds\\_%'"); + if (packet.getFieldValues().size() > 0) { + isOnRDS = true; } - }); + } catch (IOException e) { + } + } + + private TableMeta getTableMetaByDB(String fullname) throws IOException { + ResultSetPacket packet = connection.query("desc " + fullname); + String[] names = StringUtils.split(fullname, "`.`"); + String schema = names[0]; + String table = names[1].substring(0, names[1].length()); + return new TableMeta(schema, table, parserTableMeta(packet)); + } + + public static List parserTableMeta(ResultSetPacket packet) { + Map nameMaps = new HashMap(6, 1f); + + int index = 0; + for (FieldPacket fieldPacket : packet.getFieldDescriptors()) { + nameMaps.put(fieldPacket.getOriginalName(), index++); + } + + int size = packet.getFieldDescriptors().size(); + int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size(); + List result = new ArrayList(); + for (int i = 0; i < count; i++) { + FieldMeta meta = new FieldMeta(); + // 做一个优化，使用String.intern()，共享String对象，减少内存使用 + meta.setColumnName(packet.getFieldValues().get(nameMaps.get(COLUMN_NAME) + i * size).intern()); + meta.setColumnType(packet.getFieldValues().get(nameMaps.get(COLUMN_TYPE) + i * size)); + meta.setNullable(StringUtils.equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(IS_NULLABLE) + i + * size), + "YES")); + meta.setKey("PRI".equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size))); + meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size)); + meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size)); + + result.add(meta); + } + + return result; + } + + public TableMeta getTableMeta(String schema, String table) { + return getTableMeta(schema, table, true); + } + + public TableMeta getTableMeta(String schema, String table, boolean useCache) { + if (!useCache) { + tableMetaDB.invalidate(getFullName(schema, table)); + } + + return tableMetaDB.getUnchecked(getFullName(schema, table)); } public TableMeta getTableMeta(String schema, String table, BinlogPosition position) { @@ -122,20 +134,19 @@ public class TableMetaCache { tableMeta = tableMetaManager.find(schema, table); if (tableMeta == null) { // 因为条件变化，可能第一次的tableMeta没取到，需要从db获取一次，并记录到snapshot中 - String createDDL = connection.query("show create table " + getFullName(schema, table), - new ProcessJdbcResult() { - - @Override - public String process(ResultSet rs) throws SQLException { - while (rs.next()) { - return rs.getString(2); - } - return null; - } - }); - // 强制覆盖掉内存值 - tableMetaManager.apply(position, schema, createDDL); - tableMeta = tableMetaManager.find(schema, table); + String fullName = getFullName(schema, table); + try { + ResultSetPacket packet = connection.query("show create table " + fullName); + String createDDL = null; + if (packet.getFieldValues().size() > 0) { + createDDL = packet.getFieldValues().get(1); + } + // 强制覆盖掉内存值 + tableMetaManager.apply(position, schema, createDDL); + tableMeta = tableMetaManager.find(schema, table); + } catch (IOException e) { + throw new CanalParseException("fetch failed by table meta:" + fullName, e); + } } return tableMeta; } else { @@ -147,8 +158,6 @@ public class TableMetaCache { } } - - public void clearTableMeta(String schema, String table) { if (tableMetaManager != null) { // tsdb不需要做,会基于ddl sql自动清理 @@ -195,8 +204,6 @@ public class TableMetaCache { } } - - private String getFullName(String schema, String table) { StringBuilder builder = new StringBuilder(); return builder.append('`') diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java index 44975ffb..9a7a5bb5 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DdlResult.java @@ -1,6 +1,6 @@ package com.alibaba.otter.canal.parse.inbound.mysql.ddl; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DBMSAction; +import com.alibaba.otter.canal.protocol.CanalEntry.EventType; /** * @author agapple 2017年8月1日 下午7:30:42 @@ -8,30 +8,30 @@ import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DBMSAction; */ public class DdlResult { - private String schemaName; - private String tableName; - private String oriSchemaName; // rename ddl中的源表 - private String oriTableName; // rename ddl中的目标表 - private DBMSAction type; + private String schemaName; + private String tableName; + private String oriSchemaName; // rename ddl中的源表 + private String oriTableName; // rename ddl中的目标表 + private EventType type; private DdlResult renameTableResult; // 多个rename table的存储 /* * RENAME TABLE tbl_name TO new_tbl_name [, tbl_name2 TO new_tbl_name2] ... */ - public DdlResult() { + public DdlResult(){ } - public DdlResult(String schemaName) { + public DdlResult(String schemaName){ this.schemaName = schemaName; } - public DdlResult(String schemaName, String tableName) { + public DdlResult(String schemaName, String tableName){ this.schemaName = schemaName; this.tableName = tableName; } - public DdlResult(String schemaName, String tableName, String oriSchemaName, String oriTableName) { + public DdlResult(String schemaName, String tableName, String oriSchemaName, String oriTableName){ this.schemaName = schemaName; this.tableName = tableName; this.oriSchemaName = oriSchemaName; @@ -54,11 +54,11 @@ public class DdlResult { this.tableName = tableName; } - public DBMSAction getType() { + public EventType getType() { return type; } - public void setType(DBMSAction type) { + public void setType(EventType type) { this.type = type; } @@ -93,7 +93,7 @@ public class DdlResult { result.setOriTableName(oriTableName); result.setSchemaName(schemaName); result.setTableName(tableName); - //result.setType(type); + // result.setType(type); return result; } @@ -102,13 +102,12 @@ public class DdlResult { DdlResult ddlResult = this; StringBuffer sb = new StringBuffer(); do { - sb.append(String - .format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];", - ddlResult.schemaName, - ddlResult.tableName, - ddlResult.oriSchemaName, - ddlResult.oriTableName, - ddlResult.type)); + sb.append(String.format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];", + ddlResult.schemaName, + ddlResult.tableName, + ddlResult.oriSchemaName, + ddlResult.oriTableName, + ddlResult.type)); ddlResult = ddlResult.renameTableResult; } while (ddlResult != null); return sb.toString(); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java index 647955c5..15577298 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java @@ -33,11 +33,11 @@ import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStateme import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlRenameTableStatement.Item; import com.alibaba.druid.sql.parser.ParserException; import com.alibaba.druid.util.JdbcConstants; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DBMSAction; +import com.alibaba.otter.canal.protocol.CanalEntry.EventType; /** * @author agapple 2017年7月27日 下午4:05:34 - * @since 3.2.5 + * @since 1.0.25 */ public class DruidDdlParser { @@ -48,7 +48,7 @@ public class DruidDdlParser { } catch (ParserException e) { // 可能存在一些SQL是不支持的，比如存储过程 DdlResult ddlResult = new DdlResult(); - ddlResult.setType(DBMSAction.QUERY); + ddlResult.setType(EventType.QUERY); return Arrays.asList(ddlResult); } @@ -58,7 +58,7 @@ public class DruidDdlParser { DdlResult ddlResult = new DdlResult(); SQLCreateTableStatement createTable = (SQLCreateTableStatement) statement; processName(ddlResult, schmeaName, createTable.getName(), false); - ddlResult.setType(DBMSAction.CREATE); + ddlResult.setType(EventType.CREATE); ddlResults.add(ddlResult); } else if (statement instanceof SQLAlterTableStatement) { SQLAlterTableStatement alterTable = (SQLAlterTableStatement) statement; @@ -71,30 +71,30 @@ public class DruidDdlParser { } else if (item instanceof SQLAlterTableAddIndex) { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, alterTable.getName(), false); - ddlResult.setType(DBMSAction.CINDEX); + ddlResult.setType(EventType.CINDEX); ddlResults.add(ddlResult); } else if (item instanceof SQLAlterTableDropIndex || item instanceof SQLAlterTableDropKey) { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, alterTable.getName(), false); - ddlResult.setType(DBMSAction.DINDEX); + ddlResult.setType(EventType.DINDEX); ddlResults.add(ddlResult); } else if (item instanceof SQLAlterTableAddConstraint) { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, alterTable.getName(), false); SQLConstraint constraint = ((SQLAlterTableAddConstraint) item).getConstraint(); if (constraint instanceof SQLUnique) { - ddlResult.setType(DBMSAction.CINDEX); + ddlResult.setType(EventType.CINDEX); ddlResults.add(ddlResult); } } else if (item instanceof SQLAlterTableDropConstraint) { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, alterTable.getName(), false); - ddlResult.setType(DBMSAction.DINDEX); + ddlResult.setType(EventType.DINDEX); ddlResults.add(ddlResult); } else { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, alterTable.getName(), false); - ddlResult.setType(DBMSAction.ALTER); + ddlResult.setType(EventType.ALTER); ddlResults.add(ddlResult); } } @@ -103,7 +103,7 @@ public class DruidDdlParser { for (SQLExprTableSource tableSource : dropTable.getTableSources()) { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, tableSource.getExpr(), false); - ddlResult.setType(DBMSAction.ERASE); + ddlResult.setType(EventType.ERASE); ddlResults.add(ddlResult); } } else if (statement instanceof SQLCreateIndexStatement) { @@ -111,21 +111,21 @@ public class DruidDdlParser { SQLTableSource tableSource = createIndex.getTable(); DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, ((SQLExprTableSource) tableSource).getExpr(), false); - ddlResult.setType(DBMSAction.CINDEX); + ddlResult.setType(EventType.CINDEX); ddlResults.add(ddlResult); } else if (statement instanceof SQLDropIndexStatement) { SQLDropIndexStatement dropIndex = (SQLDropIndexStatement) statement; SQLExprTableSource tableSource = dropIndex.getTableName(); DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, tableSource.getExpr(), false); - ddlResult.setType(DBMSAction.DINDEX); + ddlResult.setType(EventType.DINDEX); ddlResults.add(ddlResult); } else if (statement instanceof SQLTruncateStatement) { SQLTruncateStatement truncate = (SQLTruncateStatement) statement; for (SQLExprTableSource tableSource : truncate.getTableSources()) { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, tableSource.getExpr(), false); - ddlResult.setType(DBMSAction.TRUNCATE); + ddlResult.setType(EventType.TRUNCATE); ddlResults.add(ddlResult); } } else if (statement instanceof MySqlRenameTableStatement) { @@ -134,28 +134,28 @@ public class DruidDdlParser { DdlResult ddlResult = new DdlResult(); processName(ddlResult, schmeaName, item.getName(), true); processName(ddlResult, schmeaName, item.getTo(), false); - ddlResult.setType(DBMSAction.RENAME); + ddlResult.setType(EventType.RENAME); ddlResults.add(ddlResult); } } else if (statement instanceof SQLInsertStatement) { DdlResult ddlResult = new DdlResult(); SQLInsertStatement insert = (SQLInsertStatement) statement; processName(ddlResult, schmeaName, insert.getTableName(), true); - ddlResult.setType(DBMSAction.INSERT); + ddlResult.setType(EventType.INSERT); ddlResults.add(ddlResult); } else if (statement instanceof SQLUpdateStatement) { DdlResult ddlResult = new DdlResult(); SQLUpdateStatement update = (SQLUpdateStatement) statement; // 拿到的表名可能为null,比如update a,b set a.id=x processName(ddlResult, schmeaName, update.getTableName(), true); - ddlResult.setType(DBMSAction.UPDATE); + ddlResult.setType(EventType.UPDATE); ddlResults.add(ddlResult); } else if (statement instanceof SQLDeleteStatement) { DdlResult ddlResult = new DdlResult(); SQLDeleteStatement delete = (SQLDeleteStatement) statement; // 拿到的表名可能为null,比如delete a,b from a where a.id = b.id processName(ddlResult, schmeaName, delete.getTableName(), true); - ddlResult.setType(DBMSAction.DELETE); + ddlResult.setType(EventType.DELETE); ddlResults.add(ddlResult); } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/SimpleDdlParser.java similarity index 73% rename from parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java rename to parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/SimpleDdlParser.java index c6342bff..53f7de82 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/SimpleDdlParser.java @@ -1,4 +1,4 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.dbsync; +package com.alibaba.otter.canal.parse.inbound.mysql.ddl; import org.apache.commons.lang.StringUtils; import org.apache.oro.text.regex.Perl5Matcher; @@ -218,102 +218,4 @@ public class SimpleDdlParser { sql = sb.toString(); } } - - public static class DdlResult { - - private String schemaName; - private String tableName; - private String oriSchemaName; // rename ddl中的源表 - private String oriTableName; // rename ddl中的目标表 - private EventType type; - private DdlResult renameTableResult; // 多个rename table的存储 - - /* - * RENAME TABLE tbl_name TO new_tbl_name [, tbl_name2 TO new_tbl_name2] - * ... - */ - - public DdlResult(){ - } - - public DdlResult(String schemaName){ - this.schemaName = schemaName; - } - - public DdlResult(String schemaName, String tableName){ - this.schemaName = schemaName; - this.tableName = tableName; - } - - public DdlResult(String schemaName, String tableName, String oriSchemaName, String oriTableName){ - this.schemaName = schemaName; - this.tableName = tableName; - this.oriSchemaName = oriSchemaName; - this.oriTableName = oriTableName; - } - - public String getSchemaName() { - return schemaName; - } - - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public EventType getType() { - return type; - } - - public void setType(EventType type) { - this.type = type; - } - - public String getOriSchemaName() { - return oriSchemaName; - } - - public void setOriSchemaName(String oriSchemaName) { - this.oriSchemaName = oriSchemaName; - } - - public String getOriTableName() { - return oriTableName; - } - - public void setOriTableName(String oriTableName) { - this.oriTableName = oriTableName; - } - - public DdlResult getRenameTableResult() { - return renameTableResult; - } - - public void setRenameTableResult(DdlResult renameTableResult) { - this.renameTableResult = renameTableResult; - } - - @Override - public String toString() { - DdlResult ddlResult = this; - StringBuffer sb = new StringBuffer(); - do { - sb.append(String.format("DdlResult [schemaName=%s , tableName=%s , oriSchemaName=%s , oriTableName=%s , type=%s ];", - ddlResult.schemaName, - ddlResult.tableName, - ddlResult.oriSchemaName, - ddlResult.oriTableName, - ddlResult.type)); - ddlResult = ddlResult.renameTableResult; - } while (ddlResult != null); - return sb.toString(); - } - } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java index 118da365..25f60ee3 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java @@ -1,43 +1,41 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; -import java.sql.SQLException; - import javax.sql.DataSource; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.vendor.MySqlExceptionSorter; +import com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker; import com.alibaba.otter.canal.parse.exception.CanalParseException; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** - * Created by wanshao - * Date: 2017/9/22 - * Time: 下午2:46 + * Created by wanshao Date: 2017/9/22 Time: 下午2:46 **/ public class DataSourceFactoryTSDB { - private static Logger logger = LoggerFactory.getLogger(DataSourceFactoryTSDB.class); - - public static DataSource getDataSource(String address, String userName, String password, - boolean enable, - String defaultDatabaseName, String url, String driverClassName) { - - DruidDataSource druidDataSource = new DruidDataSource(); - druidDataSource.setUrl(url); - druidDataSource.setUsername(userName); - druidDataSource.setPassword(password); - + public static DataSource getDataSource(String url, String userName, String password) { try { + DruidDataSource druidDataSource = new DruidDataSource(); + druidDataSource.setUrl(url); + druidDataSource.setUsername(userName); + druidDataSource.setPassword(password); + druidDataSource.setTestWhileIdle(true); + druidDataSource.setTestOnBorrow(false); + druidDataSource.setTestOnReturn(false); + druidDataSource.setNotFullTimeoutRetryCount(2); + druidDataSource.setValidConnectionCheckerClassName(MySqlValidConnectionChecker.class.getName()); + druidDataSource.setExceptionSorterClassName(MySqlExceptionSorter.class.getName()); + druidDataSource.setValidationQuery("SELECT 1"); + druidDataSource.setInitialSize(1); + druidDataSource.setMinIdle(1); + druidDataSource.setMaxActive(30); + druidDataSource.setMaxWait(10 * 1000); + druidDataSource.setTimeBetweenEvictionRunsMillis(60 * 1000); + druidDataSource.setMinEvictableIdleTimeMillis(50 * 1000); + druidDataSource.setUseUnfairLock(true); druidDataSource.init(); - } catch (SQLException e) { - logger.error("druidDataSource.init", e); - throw new CanalParseException("初始化druid dataSource出错"); + return druidDataSource; + } catch (Throwable e) { + throw new CanalParseException("init druidDataSource failed", e); } - return druidDataSource; } - - } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java index 293661f4..f3c6aeab 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -45,8 +45,8 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; public class MemoryTableMeta implements TableMetaTSDB { private Map, TableMeta> tableMetas = new ConcurrentHashMap, TableMeta>(); - private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); - private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class); + private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); + private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class); public MemoryTableMeta(Logger logger){ this.logger = logger; @@ -208,7 +208,7 @@ public class MemoryTableMeta implements TableMetaTSDB { if (sqlName instanceof SQLPropertyExpr) { SQLIdentifierExpr owner = (SQLIdentifierExpr) ((SQLPropertyExpr) sqlName).getOwner(); return DruidDdlParser.unescapeName(owner.getName()) + "." - + DruidDdlParser.unescapeName(((SQLPropertyExpr) sqlName).getName()); + + DruidDdlParser.unescapeName(((SQLPropertyExpr) sqlName).getName()); } else if (sqlName instanceof SQLIdentifierExpr) { return DruidDdlParser.unescapeName(((SQLIdentifierExpr) sqlName).getName()); } else { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java index 859152f7..9d926548 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java @@ -1,8 +1,6 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -15,58 +13,54 @@ import java.util.regex.Pattern; import javax.annotation.Resource; -import com.alibaba.druid.sql.repository.Schema; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONArray; -import com.alibaba.fastjson.JSONObject; -import com.alibaba.otter.canal.filter.CanalEventFilter; -import com.alibaba.otter.canal.parse.exception.CanalParseException; -import com.alibaba.otter.canal.parse.inbound.TableMeta; -import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.ProcessJdbcResult; -import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult; -import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; - -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.druid.sql.repository.Schema; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.otter.canal.filter.CanalEventFilter; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; +import com.alibaba.otter.canal.parse.exception.CanalParseException; +import com.alibaba.otter.canal.parse.inbound.TableMeta; +import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; +import com.taobao.tddl.dbsync.binlog.BinlogPosition; + /** - * 基于console远程管理 - * - * see internal class: CanalTableMeta , ConsoleTableMetaTSDB + * 基于console远程管理 see internal class: CanalTableMeta , ConsoleTableMetaTSDB * * @author agapple 2017年7月27日 下午10:47:55 * @since 3.2.5 */ public class TableMetaManager implements TableMetaTSDB { - private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); + private static Logger logger = LoggerFactory.getLogger(TableMetaManager.class); + private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); private static final BinlogPosition INIT_POSITION = BinlogPosition.parseFromString("0:0#-2.-1"); - private Logger logger = LoggerFactory.getLogger(TableMetaManager.class); - private String consoleDomain = null; - private int retry = 3; - private MemoryTableMeta memoryTableMeta; - private MysqlConnection connection; // 查询meta信息的链接 - private CanalEventFilter filter; - private BinlogPosition lastPosition; - private ScheduledExecutorService scheduler; + private MemoryTableMeta memoryTableMeta; + private MysqlConnection connection; // 查询meta信息的链接 + private CanalEventFilter filter; + private BinlogPosition lastPosition; + private ScheduledExecutorService scheduler; @Resource - private MetaHistoryDAO metaHistoryDAO; + private MetaHistoryDAO metaHistoryDAO; @Resource - private MetaSnapshotDAO metaSnapshotDAO; + private MetaSnapshotDAO metaSnapshotDAO; - public void init(){ + public void init() { this.memoryTableMeta = new MemoryTableMeta(logger); this.scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { @@ -91,7 +85,7 @@ public class TableMetaManager implements TableMetaTSDB { }, 24, 24, TimeUnit.SECONDS); } - public TableMetaManager() { + public TableMetaManager(){ } @@ -147,80 +141,46 @@ public class TableMetaManager implements TableMetaTSDB { * 初始化的时候dump一下表结构 */ private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter filter) { - List schemas = connection.query("show databases", new ProcessJdbcResult() { - - @Override - public List process(ResultSet rs) throws SQLException { - List schemas = new ArrayList(); - while (rs.next()) { - String schema = rs.getString(1); - if (!filter.filter(schema)) { - schemas.add(schema); - } + try { + ResultSetPacket packet = connection.query("show databases"); + List schemas = new ArrayList(); + for (String schema : packet.getFieldValues()) { + if (!filter.filter(schema)) { + schemas.add(schema); } - return schemas; - } - }); - - for (String schema : schemas) { - List tables = connection.query("show tables from `" + schema + "`", new ProcessJdbcResult() { - - @Override - public List process(ResultSet rs) throws SQLException { - List tables = new ArrayList(); - while (rs.next()) { - String table = rs.getString(1); - if (!filter.filter(table)) { - tables.add(table); - } - } - return tables; - } - }); - - StringBuilder sql = new StringBuilder(); - for (String table : tables) { - sql.append("show create table `" + schema + "`.`" + table + "`;"); } - // 使用多语句方式读取 - Statement stmt = null; - try { - stmt = connection.getConn().createStatement(); - ResultSet rs = stmt.executeQuery(sql.toString()); - boolean existMoreResult = false; - do { - if (existMoreResult) { - rs = stmt.getResultSet(); + for (String schema : schemas) { + packet = connection.query("show tables from `" + schema + "`"); + List tables = new ArrayList(); + for (String table : packet.getFieldValues()) { + if (!filter.filter(table)) { + tables.add(table); } + } - while (rs.next()) { - String oneTableCreateSql = rs.getString(2); + StringBuilder sql = new StringBuilder(); + for (String table : tables) { + sql.append("show create table `" + schema + "`.`" + table + "`;"); + } + + List packets = connection.queryMulti(sql.toString()); + for (ResultSetPacket onePacket : packets) { + if (onePacket.getFieldValues().size() > 1) { + String oneTableCreateSql = onePacket.getFieldValues().get(1); memoryTableMeta.apply(INIT_POSITION, schema, oneTableCreateSql); } - - existMoreResult = stmt.getMoreResults(); - } while (existMoreResult); - } catch (SQLException e) { - throw new CanalParseException(e); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - // ignore - } } } - } - return true; + return true; + } catch (IOException e) { + throw new CanalParseException(e); + } } private boolean applyHistoryToDB(BinlogPosition position, String schema, String ddl) { - Map content = new HashMap(); - content.put("binlogFile", position.getFileName()); content.put("binlogOffest", String.valueOf(position.getPosition())); content.put("binlogMasterId", String.valueOf(position.getMasterId())); @@ -240,26 +200,23 @@ public class TableMetaManager implements TableMetaTSDB { // content.put("extra", ""); } - for (int i = 0; i < retry; i++) { - MetaHistoryDO metaDO = new MetaHistoryDO(); - try { - BeanUtils.populate(metaDO, content); - // 会建立唯一约束,解决: - // 1. 重复的binlog file+offest - // 2. 重复的masterId+timestamp - metaHistoryDAO.insert(metaDO); - } catch (Throwable e) { - if (isUkDuplicateException(e)) { - // 忽略掉重复的位点 - logger.warn("dup apply for sql : " + ddl); - } else { - throw new RuntimeException("apply history to db failed caused by : " + e.getMessage()); - } - + MetaHistoryDO metaDO = new MetaHistoryDO(); + try { + BeanUtils.populate(metaDO, content); + // 会建立唯一约束,解决: + // 1. 重复的binlog file+offest + // 2. 重复的masterId+timestamp + metaHistoryDAO.insert(metaDO); + } catch (Throwable e) { + if (isUkDuplicateException(e)) { + // 忽略掉重复的位点 + logger.warn("dup apply for sql : " + ddl); + } else { + throw new CanalParseException("apply history to db failed caused by : " + e.getMessage()); } - return true; + } - return false; + return true; } /** @@ -290,9 +247,7 @@ public class TableMetaManager implements TableMetaTSDB { } } if (compareAll) { - Map content = new HashMap(); - content.put("binlogFile", position.getFileName()); content.put("binlogOffest", String.valueOf(position.getPosition())); content.put("binlogMasterId", String.valueOf(position.getMasterId())); @@ -302,24 +257,19 @@ public class TableMetaManager implements TableMetaTSDB { throw new RuntimeException("apply failed caused by content is empty in applySnapshotToDB"); } - for (int i = 0; i < retry; i++) { - MetaSnapshotDO snapshotDO = new MetaSnapshotDO(); - try { - BeanUtils.populate(snapshotDO, content); - metaSnapshotDAO.insert(snapshotDO); - } catch (Throwable e) { - if (isUkDuplicateException(e)) { - // 忽略掉重复的位点 - logger.warn("dup apply snapshot for data : " + snapshotDO.getData()); - } else { - throw new RuntimeException("apply failed caused by : " + e.getMessage()); - } + MetaSnapshotDO snapshotDO = new MetaSnapshotDO(); + try { + BeanUtils.populate(snapshotDO, content); + metaSnapshotDAO.insert(snapshotDO); + } catch (Throwable e) { + if (isUkDuplicateException(e)) { + // 忽略掉重复的位点 + logger.warn("dup apply snapshot for data : " + snapshotDO.getData()); + } else { + throw new CanalParseException("apply failed caused by : " + e.getMessage()); } - return true; - } - return false; - + return true; } else { logger.error("compare failed , check log"); } @@ -327,26 +277,15 @@ public class TableMetaManager implements TableMetaTSDB { } private boolean compareTableMetaDbAndMemory(MysqlConnection connection, final String schema, final String table) { - TableMeta tableMetaFromDB = connection.query("desc " + getFullName(schema, table), - new ProcessJdbcResult() { - - @Override - public TableMeta process(ResultSet rs) throws SQLException { - List metas = new ArrayList(); - while (rs.next()) { - FieldMeta meta = new FieldMeta(); - // 做一个优化，使用String.intern()，共享String对象，减少内存使用 - meta.setColumnName(rs.getString("Field")); - meta.setColumnType(rs.getString("Type")); - meta.setNullable(StringUtils.equalsIgnoreCase(rs.getString("Null"), "YES")); - meta.setKey("PRI".equalsIgnoreCase(rs.getString("Key"))); - meta.setDefaultValue(rs.getString("Default")); - metas.add(meta); - } - - return new TableMeta(schema, table, metas); - } - }); + TableMeta tableMetaFromDB = new TableMeta(); + tableMetaFromDB.setSchema(schema); + tableMetaFromDB.setTable(table); + try { + ResultSetPacket packet = connection.query("desc " + getFullName(schema, table)); + tableMetaFromDB.setFields(TableMetaCache.parserTableMeta(packet)); + } catch (IOException e) { + throw new CanalParseException(e); + } TableMeta tableMetaFromMem = memoryTableMeta.find(schema, table); boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB); @@ -358,119 +297,70 @@ public class TableMetaManager implements TableMetaTSDB { } private BinlogPosition buildMemFromSnapshot(BinlogPosition position) { + try { + MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(position.getTimestamp()); + String binlogFile = snapshotDO.getBinlogFile(); + Long binlogOffest = snapshotDO.getBinlogOffest(); + String binlogMasterId = snapshotDO.getBinlogMasterId(); + Long binlogTimestamp = snapshotDO.getBinlogTimestamp(); - Map content = new HashMap(); - - content.put("binlogFile", position.getFileName()); - content.put("binlogOffest", String.valueOf(position.getPosition())); - content.put("binlogMasterId", String.valueOf(position.getMasterId())); - content.put("binlogTimestamp", String.valueOf(position.getTimestamp())); - if (content.isEmpty()) { - throw new RuntimeException("apply failed caused by content is empty in buildMemFromSnapshot"); - } - for (int i = 0; i < retry; i++) { - - try { - - String timestamp = content.get("binlogTimestamp"); - MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(Long.valueOf(timestamp)); - JSONObject jsonData = new JSONObject(); - jsonData.put("content", JSON.toJSONString(snapshotDO)); - if (jsonData == null) { - // 可能没有任何snapshot数据 + BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, + binlogOffest == null ? 0l : binlogOffest, + Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), + binlogTimestamp == null ? 0l : binlogTimestamp); + // data存储为Map，每个分库一套建表 + String sqlData = snapshotDO.getData(); + JSONObject jsonObj = JSON.parseObject(sqlData); + for (Map.Entry entry : jsonObj.entrySet()) { + // 记录到内存 + if (!memoryTableMeta.apply(snapshotPosition, + ObjectUtils.toString(entry.getKey()), + ObjectUtils.toString(entry.getValue()))) { return null; } - - String binlogFile = jsonData.getString("binlogFile"); - String binlogOffest = jsonData.getString("binlogOffest"); - String binlogMasterId = jsonData.getString("binlogMasterId"); - String binlogTimestamp = jsonData.getString("binlogTimestamp"); - - BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, - Long.valueOf(binlogOffest == null ? "0" : binlogOffest), - Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), - Long.valueOf(binlogTimestamp == null ? "0" : binlogTimestamp)); - // data存储为Map，每个分库一套建表 - String sqlData = jsonData.getString("data"); - JSONObject jsonObj = JSON.parseObject(sqlData); - for (Map.Entry entry : jsonObj.entrySet()) { - // 记录到内存 - if (!memoryTableMeta.apply(snapshotPosition, - ObjectUtils.toString(entry.getKey()), - ObjectUtils.toString(entry.getValue()))) { - return null; - } - } - - return snapshotPosition; - - } catch (Throwable e) { - throw new RuntimeException("apply failed caused by : " + e.getMessage()); } + return snapshotPosition; + } catch (Throwable e) { + throw new CanalParseException("apply failed caused by : " + e.getMessage()); } - - return null; - } private boolean applyHistoryOnMemory(BinlogPosition position, BinlogPosition rollbackPosition) { - Map content = new HashMap(); - content.put("binlogSnapshotTimestamp", String.valueOf(position.getTimestamp())); - content.put("binlogFile", rollbackPosition.getFileName()); - content.put("binlogOffest", String.valueOf(rollbackPosition.getPosition())); - content.put("binlogMasterId", String.valueOf(rollbackPosition.getMasterId())); - content.put("binlogTimestamp", String.valueOf(rollbackPosition.getTimestamp())); - String timestamp = content.get("binlogTimestamp"); - String binlogSnapshotTimestamp = content.get("binlogSnapshotTimestamp"); - - for (int i = 0; i < retry; i++) { - try { - List metaHistoryDOList = metaHistoryDAO.findByTimestamp( - Long.valueOf(binlogSnapshotTimestamp), - Long.valueOf(timestamp)); - JSONObject json = new JSONObject(); - json.put("content", JSON.toJSONString(metaHistoryDOList)); - String data = ObjectUtils.toString(json.get("content")); - JSONArray jsonArray = JSON.parseArray(data); - for (Object jsonObj : jsonArray) { - JSONObject jsonData = (JSONObject)jsonObj; - String binlogFile = jsonData.getString("binlogFile"); - String binlogOffest = jsonData.getString("binlogOffest"); - String binlogMasterId = jsonData.getString("binlogMasterId"); - String binlogTimestamp = jsonData.getString("binlogTimestamp"); - String useSchema = jsonData.getString("useSchema"); - String sqlData = jsonData.getString("sql"); - BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, - Long.valueOf(binlogOffest == null ? "0" : binlogOffest), - Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), - Long.valueOf(binlogTimestamp == null ? "0" : binlogTimestamp)); - - // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之 - if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) { - continue; - } else if (rollbackPosition.getMasterId() == snapshotPosition.getMasterId() - && snapshotPosition.compareTo(rollbackPosition) > 0) { - continue; - } - - // 记录到内存 - if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData)) { - return false; - } + try { + List metaHistoryDOList = metaHistoryDAO.findByTimestamp(position.getTimestamp(), + rollbackPosition.getTimestamp()); + for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) { + String binlogFile = metaHistoryDO.getBinlogFile(); + Long binlogOffest = metaHistoryDO.getBinlogOffest(); + String binlogMasterId = metaHistoryDO.getBinlogMasterId(); + Long binlogTimestamp = metaHistoryDO.getBinlogTimestamp(); + String useSchema = metaHistoryDO.getUseSchema(); + String sqlData = metaHistoryDO.getSql(); + BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, + binlogOffest == null ? 0L : binlogOffest, + Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), + binlogTimestamp == null ? 0L : binlogTimestamp); + // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之 + if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) { + continue; + } else if (rollbackPosition.getMasterId() == snapshotPosition.getMasterId() + && snapshotPosition.compareTo(rollbackPosition) > 0) { + continue; } - return jsonArray.size() > 0; - } catch (Throwable e) { - - throw new RuntimeException("apply failed", e); + // 记录到内存 + if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData)) { + return false; + } } + return metaHistoryDOList.size() > 0; + } catch (Throwable e) { + throw new CanalParseException("apply failed", e); } - - return false; } private String getFullName(String schema, String table) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java index 054a4b44..46ffafbf 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java @@ -3,18 +3,17 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; import java.util.HashMap; import java.util.List; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; - -import com.google.common.collect.Maps; import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; +import com.google.common.collect.Maps; + /** * canal数据的存储 * * @author wanshao 2017年7月27日 下午10:51:55 * @since 3.2.5 */ -@SuppressWarnings("deprecation") public class MetaHistoryDAO extends SqlMapClientDaoSupport { public List getAll() { @@ -22,14 +21,14 @@ public class MetaHistoryDAO extends SqlMapClientDaoSupport { } public Long insert(MetaHistoryDO metaDO) { - return (Long)getSqlMapClientTemplate().insert("table_meta_history.insert", metaDO); + return (Long) getSqlMapClientTemplate().insert("table_meta_history.insert", metaDO); } public List findByTimestamp(long snapshotTimestamp, long timestamp) { HashMap params = Maps.newHashMapWithExpectedSize(2); params.put("snapshotTimestamp", snapshotTimestamp); params.put("timestamp", timestamp); - return (List)getSqlMapClientTemplate().queryForList("table_meta_history.findByTimestamp", + return (List) getSqlMapClientTemplate().queryForList("table_meta_history.findByTimestamp", params); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java index 79f1f16f..ca32ebd3 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java @@ -2,11 +2,11 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; import java.util.HashMap; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; - -import com.google.common.collect.Maps; import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; +import com.google.common.collect.Maps; + /** * canal数据的存储 * @@ -15,6 +15,7 @@ import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; */ public class MetaSnapshotDAO extends SqlMapClientDaoSupport { + public Long insert(MetaSnapshotDO snapshotDO) { return (Long) getSqlMapClientTemplate().insert("table_meta_snapshot.insert", snapshotDO); } @@ -26,8 +27,7 @@ public class MetaSnapshotDAO extends SqlMapClientDaoSupport { public MetaSnapshotDO findByTimestamp(long timestamp) { HashMap params = Maps.newHashMapWithExpectedSize(2); params.put("timestamp", timestamp); - return (MetaSnapshotDO) getSqlMapClientTemplate().queryForObject("table_meta_snapshot.findByTimestamp", - params); + return (MetaSnapshotDO) getSqlMapClientTemplate().queryForObject("table_meta_snapshot.findByTimestamp", params); } public Integer deleteByTask(String taskName) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java index 9b56606d..96108515 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java @@ -11,22 +11,22 @@ public class MetaHistoryDO { /** * 主键 */ - private Long id; + private Long id; /** * 创建时间 */ - private Date gmtCreate; + private Date gmtCreate; /** * 修改时间 */ - private Date gmtModified; + private Date gmtModified; private String binlogFile; - private Long binlogOffest; + private Long binlogOffest; private String binlogMasterId; - private Long binlogTimestamp; + private Long binlogTimestamp; private String useSchema; private String schema; private String table; diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java index 4652fe13..8ddfbc64 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java @@ -11,22 +11,22 @@ public class MetaSnapshotDO { /** * 主键 */ - private Long id; + private Long id; /** * 创建时间 */ - private Date gmtCreate; + private Date gmtCreate; /** * 修改时间 */ - private Date gmtModified; + private Date gmtModified; private String binlogFile; - private Long binlogOffest; + private Long binlogOffest; private String binlogMasterId; - private Long binlogTimestamp; + private Long binlogTimestamp; private String data; private String extra; diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java index ed0b9879..bcc2af7d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java @@ -45,6 +45,7 @@ public class FileMixedLogPositionManager extends AbstractLogPositionManager { private ScheduledExecutorService executorService; + @SuppressWarnings("serial") private final LogPosition nullPosition = new LogPosition() { }; diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java index 4a1e60c7..6dbda0a5 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java @@ -29,6 +29,7 @@ public class PeriodMixedLogPositionManager extends AbstractLogPositionManager { private long period; private Set persistTasks; + @SuppressWarnings("serial") private final LogPosition nullPosition = new LogPosition() { }; diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java index 0c09dc0b..8a78e246 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java @@ -2,34 +2,43 @@ package com.alibaba.otter.canal.parse.inbound; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.List; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.junit.Assert; import org.junit.Test; -import com.alibaba.druid.sql.visitor.functions.Bin; -import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; public class TableMetaCacheTest { - //@Test - //public void testSimple() { - // - // MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"); - // try { - // connection.connect(); - // } catch (IOException e) { - // Assert.fail(e.getMessage()); - // } - // - // TableMetaCache cache = new TableMetaCache(connection); - // TableMeta meta = cache.getTableMeta("otter1", "otter_stability1"); - // Assert.assertNotNull(meta); - // for (FieldMeta field : meta.getFields()) { - // System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : " - // + field.isNullable()); - // } - //} + @Test + public void testSimple() throws IOException { + + MysqlConnection connection = new MysqlConnection(new InetSocketAddress("100.81.153.8", 3308), + "jingwei", + "jingwei"); + try { + connection.connect(); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + + List packets = connection.queryMulti("show create table test.ljh_test; show create table test.user"); + String createDDL = null; + if (packets.get(0).getFieldValues().size() > 0) { + createDDL = packets.get(0).getFieldValues().get(1); + } + + System.out.println(createDDL); + + // TableMetaCache cache = new TableMetaCache(connection); + // TableMeta meta = cache.getTableMeta("otter1", "otter_stability1"); + // Assert.assertNotNull(meta); + // for (FieldMeta field : meta.getFields()) { + // System.out.println("filed :" + field.getColumnName() + " , isKey : " + // + field.isKey() + " , isNull : " + // + field.isNullable()); + // } + } } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java index 5d3e8920..57e26784 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java @@ -88,7 +88,6 @@ public class GroupEventPaserTest { private BinlogParser buildParser(AuthenticationInfo info) { return new AbstractBinlogParser() { - @Override public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException { return null; diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java index 50dce482..f2bb1ce4 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java @@ -3,8 +3,8 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import org.junit.Assert; import org.junit.Test; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser; -import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.SimpleDdlParser; import com.alibaba.otter.canal.protocol.CanalEntry.EventType; public class SimpleDdlParserTest { diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java index c928b667..8ead5e2a 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java @@ -19,7 +19,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; * @author agapple 2017年8月1日 下午7:15:54 */ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = {"/dal-dao.xml"}) +@ContextConfiguration(locations = { "/dal-dao.xml" }) public class MemoryTableMetaTest { @Test @@ -33,6 +33,6 @@ public class MemoryTableMetaTest { TableMeta meta = memoryTableMeta.find("test", "test"); System.out.println(meta); - Assert.assertTrue( meta.getFieldMetaByName("ID").isKey()); + Assert.assertTrue(meta.getFieldMetaByName("ID").isKey()); } } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java index 30ae1e66..29c02f7e 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java @@ -13,12 +13,10 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; /** - * Created by wanshao - * Date: 2017/9/20 - * Time: 下午5:00 + * Created by wanshao Date: 2017/9/20 Time: 下午5:00 **/ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = {"/dal-dao.xml"}) +@ContextConfiguration(locations = { "/dal-dao.xml" }) public class MetaHistoryDAOTest { @Resource @@ -26,8 +24,7 @@ public class MetaHistoryDAOTest { @Test public void testGetAll() { - List - metaHistoryDOList = metaHistoryDAO.getAll(); + List metaHistoryDOList = metaHistoryDAO.getAll(); for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) { System.out.println(metaHistoryDO.getId()); } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java index 1eb45d7d..13413d81 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java @@ -30,8 +30,6 @@ public class TableMetaManagerTest { @Test public void testSimple() throws FileNotFoundException, IOException { - - URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); File dummyFile = new File(url.getFile()); File create = new File(dummyFile.getParent() + "/ddl", "create.sql"); diff --git a/parse/src/test/resources/dal-dao.xml b/parse/src/test/resources/dal-dao.xml deleted file mode 100644 index 7d2fd0a4..00000000 --- a/parse/src/test/resources/dal-dao.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/parse/src/main/resources/dal-dao.xml b/parse/src/test/resources/tsdb/dal-dao.xml similarity index 83% rename from parse/src/main/resources/dal-dao.xml rename to parse/src/test/resources/tsdb/dal-dao.xml index 942de654..1842641f 100644 --- a/parse/src/main/resources/dal-dao.xml +++ b/parse/src/test/resources/tsdb/dal-dao.xml @@ -9,13 +9,10 @@ - + - - - @@ -32,11 +29,11 @@ - + - + diff --git a/parse/src/main/resources/mybatis-config.xml b/parse/src/test/resources/tsdb/mybatis-config.xml similarity index 100% rename from parse/src/main/resources/mybatis-config.xml rename to parse/src/test/resources/tsdb/mybatis-config.xml diff --git a/deployer/src/main/resources/spring/sql-map/sqlmap_history.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml similarity index 100% rename from deployer/src/main/resources/spring/sql-map/sqlmap_history.xml rename to parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml diff --git a/parse/src/main/resources/sql-map/sqlmap_snapshot.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml similarity index 100% rename from parse/src/main/resources/sql-map/sqlmap_snapshot.xml rename to parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml diff --git a/parse/src/main/resources/sqlmap-config.xml b/parse/src/test/resources/tsdb/sqlmap-config.xml similarity index 100% rename from parse/src/main/resources/sqlmap-config.xml rename to parse/src/test/resources/tsdb/sqlmap-config.xml From 4f00f5ea8da0ca78b1744376d2b08dd26a7e712b Mon Sep 17 00:00:00 2001 From: agapple Date: Sat, 14 Oct 2017 00:12:43 +0800 Subject: [PATCH 03/34] fixed tsdb test --- common/pom.xml | 34 +++- .../tddl/dbsync/binlog/BinlogPosition.java | 142 ---------------- deployer/src/main/resources/canal.properties | 3 + .../resources/example/instance.properties | 14 +- .../resources/spring/default-instance.xml | 5 + .../main/resources/spring/file-instance.xml | 40 +---- .../main/resources/spring/group-instance.xml | 2 + .../main/resources/spring/local-instance.xml | 5 + .../main/resources/spring/memory-instance.xml | 5 + .../main/resources/spring/tsdb/dal-dao.xml | 39 ----- .../resources/spring/tsdb/mybatis-config.xml | 20 --- .../main/resources/spring/tsdb/mysql-tsdb.xml | 63 +++++++ .../tsdb/{ => sql-map}/sqlmap-config.xml | 5 +- .../spring/tsdb/sql-map/sqlmap_history.xml | 38 ++--- .../spring/tsdb/sql-map/sqlmap_snapshot.xml | 53 +++--- .../spring/tsdb/sql/create_table.sql | 39 +++++ .../driver/mysql/socket/SocketChannel.java | 11 +- .../canal/meta/FileMixedMetaManager.java | 2 +- .../canal/meta/FileMixedMetaManagerTest.java | 10 +- parse/pom.xml | 4 +- .../parse/inbound/AbstractEventParser.java | 10 +- .../mysql/AbstractMysqlEventParser.java | 71 +++++--- .../inbound/mysql/LocalBinlogEventParser.java | 2 +- .../parse/inbound/mysql/MysqlEventParser.java | 51 +++++- .../inbound/mysql/dbsync/LogEventConvert.java | 19 ++- .../inbound/mysql/dbsync/TableMetaCache.java | 36 ++-- .../mysql/tsdb/DataSourceFactoryTSDB.java | 41 ----- .../inbound/mysql/tsdb/MemoryTableMeta.java | 33 +++- .../inbound/mysql/tsdb/TableMetaManager.java | 160 ++++++++++-------- .../inbound/mysql/tsdb/TableMetaTSDB.java | 14 +- .../mysql/tsdb/TableMetaTSDBBuilder.java | 51 ++++++ .../mysql/tsdb/dao/MetaHistoryDAO.java | 65 +++++-- .../tsdb/{model => dao}/MetaHistoryDO.java | 105 ++++++------ .../mysql/tsdb/dao/MetaSnapshotDAO.java | 61 +++++-- .../tsdb/{model => dao}/MetaSnapshotDO.java | 33 ++-- .../main/resources/ddl/derby/meta_history.sql | 22 +++ .../resources/ddl/derby/meta_snapshot.sql | 18 ++ .../main/resources/ddl/mysql/meta_history.sql | 21 +++ .../resources/ddl/mysql/meta_snapshot.sql | 17 ++ .../mysql/LocalBinlogEventParserTest.java | 2 +- .../parse/inbound/mysql/MysqlDumpTest.java | 10 +- .../mysql/tsdb/MemoryTableMetaTest.java | 11 +- .../mysql/tsdb/MetaHistoryDAOTest.java | 12 +- .../tsdb/TableMetaManagerBuilderTest.java | 18 ++ .../mysql/tsdb/TableMetaManagerTest.java | 14 +- parse/src/test/resources/ddl/create.sql | 2 +- parse/src/test/resources/tsdb/dal-dao.xml | 39 ----- parse/src/test/resources/tsdb/derby-tsdb.xml | 44 +++++ .../test/resources/tsdb/mybatis-config.xml | 20 --- parse/src/test/resources/tsdb/mysql-tsdb.xml | 47 +++++ .../tsdb/{ => sql-map}/sqlmap-config.xml | 5 +- .../resources/tsdb/sql-map/sqlmap_history.xml | 44 ++--- .../tsdb/sql-map/sqlmap_snapshot.xml | 63 ++++--- pom.xml | 34 +++- .../protocol/position/EntryPosition.java | 15 ++ 55 files changed, 1002 insertions(+), 742 deletions(-) delete mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java delete mode 100644 deployer/src/main/resources/spring/tsdb/dal-dao.xml delete mode 100644 deployer/src/main/resources/spring/tsdb/mybatis-config.xml create mode 100644 deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml rename deployer/src/main/resources/spring/tsdb/{ => sql-map}/sqlmap-config.xml (66%) create mode 100644 deployer/src/main/resources/spring/tsdb/sql/create_table.sql delete mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java rename parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/{model => dao}/MetaHistoryDO.java (55%) rename parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/{model => dao}/MetaSnapshotDO.java (72%) create mode 100644 parse/src/main/resources/ddl/derby/meta_history.sql create mode 100644 parse/src/main/resources/ddl/derby/meta_snapshot.sql create mode 100644 parse/src/main/resources/ddl/mysql/meta_history.sql create mode 100644 parse/src/main/resources/ddl/mysql/meta_snapshot.sql create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java delete mode 100644 parse/src/test/resources/tsdb/dal-dao.xml create mode 100644 parse/src/test/resources/tsdb/derby-tsdb.xml delete mode 100644 parse/src/test/resources/tsdb/mybatis-config.xml create mode 100644 parse/src/test/resources/tsdb/mysql-tsdb.xml rename parse/src/test/resources/tsdb/{ => sql-map}/sqlmap-config.xml (68%) diff --git a/common/pom.xml b/common/pom.xml index 6a6df57b..4789ac9e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -1,4 +1,5 @@ - + 4.0.0 com.alibaba.otter @@ -34,10 +35,6 @@ commons-lang commons-lang - - org.springframework - spring - com.alibaba fastjson @@ -63,10 +60,37 @@ org.slf4j slf4j-api + + org.springframework + spring-core + + + org.springframework + spring-aop + + + org.springframework + spring-context + + + org.springframework + spring-jdbc + + + org.springframework + spring-orm + + + org.springframework + spring-test + ${spring_version} + test + junit junit + test diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java deleted file mode 100644 index a602c05f..00000000 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/BinlogPosition.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.taobao.tddl.dbsync.binlog; - - -/** - * Position inside binlog file - * - * @author Seppo Jaakola - * @version 1.0 - */ -public class BinlogPosition extends LogPosition { - - /* The source server_id of position, 0 invalid */ - protected final long masterId; - - /* The timestamp, in seconds, 0 invalid */ - protected final long timestamp; - - public BinlogPosition(String fileName, long position, long masterId, long timestamp){ - super(fileName, position); - this.masterId = masterId; - this.timestamp = timestamp; - } - - public BinlogPosition(LogPosition logPosition, long masterId, long timestamp){ - super(logPosition.getFileName(), logPosition.getPosition()); - this.masterId = masterId; - this.timestamp = timestamp; - } - - public BinlogPosition(BinlogPosition binlogPosition){ - super(binlogPosition.getFileName(), binlogPosition.getPosition()); - this.masterId = binlogPosition.masterId; - this.timestamp = binlogPosition.timestamp; - } - - private final static long[] pow10 = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, - 10000000000L, 100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, - 10000000000000000L, 100000000000000000L, 1000000000000000000L }; - - public static String placeHolder(int bit, long number) { - if (bit > 18) { - throw new IllegalArgumentException("Bit must less than 18, but given " + bit); - } - - final long max = pow10[bit]; - if (number >= max) { - // 当 width < 数值的最大位数时，应该直接返回数值 - return String.valueOf(number); - } - - return String.valueOf(max + number).substring(1); - } - - public String format2String(final int positionMaxLen) { - String binlogSuffix = fileName; - String binlogOffset = placeHolder((int) positionMaxLen, position); - // 输出 '000001:0000000004@12+12314130' - StringBuffer buf = new StringBuffer(40); - buf.append(binlogSuffix); - buf.append(':'); - buf.append(binlogOffset); - if (masterId != 0) { - buf.append('#'); - buf.append(masterId); - } - if (timestamp != 0) { - buf.append('.'); - buf.append(timestamp); - } - return buf.toString(); - } - - public static BinlogPosition parseFromString(String source) { - int colonIndex = source.indexOf(':'); - int miscIndex = colonIndex + 1; - int sharpIndex = source.indexOf('#', miscIndex); - int semicolonIndex = source.indexOf(';', miscIndex); // NOTE: 向后兼容 - int dotIndex = source.lastIndexOf('.'); - if (colonIndex == -1) { - return null; // NOTE: 错误的位点 - } - - String binlogSuffix = source.substring(0, colonIndex); - long binlogPosition; - if (sharpIndex != -1) { - binlogPosition = Long.parseLong(source.substring(miscIndex, sharpIndex)); - } else if (semicolonIndex != -1) { - binlogPosition = Long.parseLong(source.substring(miscIndex, semicolonIndex)); // NOTE: - // 向后兼容 - } else if (dotIndex != -1) { - binlogPosition = Long.parseLong(source.substring(miscIndex, dotIndex)); - } else { - binlogPosition = Long.parseLong(source.substring(miscIndex)); - } - - long masterId = 0; // NOTE: 默认值为 0 - if (sharpIndex != -1) { - if (dotIndex != -1) { - masterId = Long.parseLong(source.substring(sharpIndex + 1, dotIndex)); - } else { - masterId = Long.parseLong(source.substring(sharpIndex + 1)); - } - } - - long timestamp = 0; // NOTE: 默认值为 0 - if (dotIndex != -1 && dotIndex > colonIndex) { - timestamp = Long.parseLong(source.substring(dotIndex + 1)); - } - - return new BinlogPosition(binlogSuffix, binlogPosition, // NL - masterId, - timestamp); - } - - public String getFilePattern() { - final int index = fileName.indexOf('.'); - if (index != -1) { - return fileName.substring(0, index); - } - return null; - } - - public void setFilePattern(String filePattern) { - // We tolerate the event ID with or without the binlog prefix. - if (fileName.indexOf('.') < 0) { - fileName = filePattern + '.' + fileName; - } - } - - public long getMasterId() { - return masterId; - } - - public long getTimestamp() { - return timestamp; - } - - @Override - public String toString() { - return format2String(10); - } -} diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index df6f49e2..09616194 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -36,6 +36,7 @@ canal.instance.network.sendBufferSize = 16384 canal.instance.network.soTimeout = 30 # binlog filter config +canal.instance.filter.druid.ddl = true canal.instance.filter.query.dcl = false canal.instance.filter.query.dml = false canal.instance.filter.query.ddl = false @@ -59,6 +60,8 @@ canal.conf.dir = ../conf canal.auto.scan = true canal.auto.scan.interval = 5 +canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml + canal.instance.global.mode = spring canal.instance.global.lazy = false #canal.instance.global.manager.address = 127.0.0.1:1099 diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 7f00f079..50640d99 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -3,14 +3,14 @@ canal.instance.mysql.slaveId=1234 # position info canal.instance.master.address=127.0.0.1:3306 -canal.instance.master.journal.name=mysql-bin.000004 -canal.instance.master.position=25678 -canal.instance.master.timestamp=1506088042 +canal.instance.master.journal.name=mysql-bin.000001 +canal.instance.master.position=68240 +canal.instance.master.timestamp= -# tsdb info +# table meta tsdb info canal.instance.tsdb.enable=true -canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/tsdb +canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb canal.instance.tsdb.dbUsername=canal canal.instance.tsdb.dbPassword=canal @@ -25,7 +25,7 @@ canal.instance.dbPassword=canal canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 # table regex -canal.instance.filter.regex=.*\\..* +canal.instance.filter.regex=test\\..* # table black regex -canal.instance.filter.black.regex=tsdb\\..* +canal.instance.filter.black.regex=canal_tsdb\\..* ################################################# \ No newline at end of file diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index 5631b7cb..9c7ef5aa 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -178,9 +178,14 @@ + + + + + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index a9f2823b..4c6f750d 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -163,48 +163,14 @@ + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index 6fefaa87..f5e872f3 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -160,6 +160,7 @@ + @@ -250,6 +251,7 @@ + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index e5e4e695..e2fb44b3 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -155,9 +155,14 @@ + + + + + diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index 269b9deb..129ce2b0 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -151,9 +151,14 @@ + + + + + diff --git a/deployer/src/main/resources/spring/tsdb/dal-dao.xml b/deployer/src/main/resources/spring/tsdb/dal-dao.xml deleted file mode 100644 index 1842641f..00000000 --- a/deployer/src/main/resources/spring/tsdb/dal-dao.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/deployer/src/main/resources/spring/tsdb/mybatis-config.xml b/deployer/src/main/resources/spring/tsdb/mybatis-config.xml deleted file mode 100644 index 83ac0116..00000000 --- a/deployer/src/main/resources/spring/tsdb/mybatis-config.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml new file mode 100644 index 00000000..5f02431d --- /dev/null +++ b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml @@ -0,0 +1,63 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deployer/src/main/resources/spring/tsdb/sqlmap-config.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml similarity index 66% rename from deployer/src/main/resources/spring/tsdb/sqlmap-config.xml rename to deployer/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml index 983e9663..26690703 100644 --- a/deployer/src/main/resources/spring/tsdb/sqlmap-config.xml +++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap-config.xml @@ -3,7 +3,6 @@ "http://www.ibatis.com/dtd/sql-map-config-2.dtd"> - - - + + diff --git a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml index a983e222..7445bf69 100644 --- a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml +++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_history.xml @@ -1,51 +1,45 @@ - - - - + + - insert into `canal_table_meta_history` () - values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#schema#,#table#,#sql#,#type#,#extra#); - - SELECT last_insert_id() - + insert into meta_history () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#sqlSchema#,#sqlTable#,#sqlText#,#sqlType#,#extra#) + + + delete from meta_history + where destination=#destination# + - - - - \ No newline at end of file diff --git a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml index 9065a7e0..79aa25fe 100644 --- a/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml +++ b/deployer/src/main/resources/spring/tsdb/sql-map/sqlmap_snapshot.xml @@ -1,66 +1,51 @@ - - - - - - + + - + - insert into `canal_table_meta_snapshot` () - values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#); - - SELECT last_insert_id() - + insert into meta_snapshot () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#) - update `canal_table_meta_snapshot` set gmt_modified=now(), + update meta_snapshot set gmt_modified=now(), binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra# - where binlog_timestamp=0 + where destination=#destination# and binlog_timestamp=0 + + delete from meta_snapshot + where destination=#destination# + - - - \ No newline at end of file diff --git a/deployer/src/main/resources/spring/tsdb/sql/create_table.sql b/deployer/src/main/resources/spring/tsdb/sql/create_table.sql new file mode 100644 index 00000000..e5d6dbf4 --- /dev/null +++ b/deployer/src/main/resources/spring/tsdb/sql/create_table.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS `meta_snapshot` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `data` longtext DEFAULT NULL COMMENT '表结构数据', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表'; + +CREATE TABLE IF NOT EXISTS `meta_history` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', + `schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', + `table` varchar(1024) DEFAULT NULL COMMENT '对应的table', + `sql` longtext DEFAULT NULL COMMENT '执行的sql', + `type` varchar(256) DEFAULT NULL COMMENT 'sql类型', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表'; \ No newline at end of file diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java index e825aa2b..a4568887 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java @@ -35,21 +35,24 @@ public class SocketChannel { } public void writeChannel(byte[]... buf) throws IOException { - if (channel != null && channel.isWritable()) channel.writeAndFlush(Unpooled.copiedBuffer(buf)); - else throw new IOException("write failed ! please checking !"); + if (channel != null && channel.isWritable()) { + channel.writeAndFlush(Unpooled.copiedBuffer(buf)); + } else { + throw new IOException("write failed ! please checking !"); + } } public byte[] read(int readSize) throws IOException { do { if (readSize > cache.readableBytes()) { if (null == channel) { - throw new IOException("socket has Interrupted !"); + throw new java.nio.channels.ClosedByInterruptException(); } synchronized (this) { try { wait(100); } catch (InterruptedException e) { - throw new IOException("socket has Interrupted !"); + throw new java.nio.channels.ClosedByInterruptException(); } } } else { diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java index 72e3646b..383fb351 100644 --- a/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java +++ b/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java @@ -367,7 +367,7 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta this.dataDir = new File(dataDir); } - public void setDataDir(File dataDir) { + public void setDataDirByFile(File dataDir) { this.dataDir = dataDir; } diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java index 56ea51a8..4c330c86 100644 --- a/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java +++ b/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java @@ -31,7 +31,7 @@ public class FileMixedMetaManagerTest extends AbstractMetaManagerTest { @Test public void testSubscribeAll() { FileMixedMetaManager metaManager = new FileMixedMetaManager(); - metaManager.setDataDir(dataDir); + metaManager.setDataDirByFile(dataDir); metaManager.setPeriod(100); metaManager.start(); @@ -40,7 +40,7 @@ public class FileMixedMetaManagerTest extends AbstractMetaManagerTest { sleep(2000L); // 重新构建一次，能获得上一次zk上的记录 FileMixedMetaManager metaManager2 = new FileMixedMetaManager(); - metaManager2.setDataDir(dataDir); + metaManager2.setDataDirByFile(dataDir); metaManager2.setPeriod(100); metaManager2.start(); @@ -52,7 +52,7 @@ public class FileMixedMetaManagerTest extends AbstractMetaManagerTest { @Test public void testBatchAll() { FileMixedMetaManager metaManager = new FileMixedMetaManager(); - metaManager.setDataDir(dataDir); + metaManager.setDataDirByFile(dataDir); metaManager.setPeriod(100); metaManager.start(); @@ -67,7 +67,7 @@ public class FileMixedMetaManagerTest extends AbstractMetaManagerTest { @Test public void testCursorAll() { FileMixedMetaManager metaManager = new FileMixedMetaManager(); - metaManager.setDataDir(dataDir); + metaManager.setDataDirByFile(dataDir); metaManager.setPeriod(100); metaManager.start(); @@ -76,7 +76,7 @@ public class FileMixedMetaManagerTest extends AbstractMetaManagerTest { sleep(1000L); // 重新构建一次，能获得上一次zk上的记录 FileMixedMetaManager metaManager2 = new FileMixedMetaManager(); - metaManager2.setDataDir(dataDir); + metaManager2.setDataDirByFile(dataDir); metaManager2.setPeriod(100); metaManager2.start(); diff --git a/parse/pom.xml b/parse/pom.xml index 0adf86e0..66e4bb60 100644 --- a/parse/pom.xml +++ b/parse/pom.xml @@ -1,4 +1,5 @@ - + 4.0.0 com.alibaba.otter @@ -56,7 +57,6 @@ org.apache.ibatis ibatis-sqlmap - 2.3.4.726 diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java index 32d3aee5..1ea6af96 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java @@ -96,6 +96,10 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle protected void preDump(ErosaConnection connection) { } + protected boolean processTableMeta(EntryPosition position) { + return true; + } + protected void afterDump(ErosaConnection connection) { } @@ -145,7 +149,6 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle ErosaConnection erosaConnection = null; while (running) { try { - // 开始执行replication // 1. 构造Erosa连接 erosaConnection = buildErosaConnection(); @@ -163,6 +166,11 @@ public abstract class AbstractEventParser extends AbstractCanalLifeCycle if (startPosition == null) { throw new CanalParseException("can't find start position for " + destination); } + + if (!processTableMeta(startPosition)) { + throw new CanalParseException("can't find init table meta for " + destination + + " with position : " + startPosition); + } logger.info("find start position : {}", startPosition.toString()); // 重新链接，因为在找position过程中可能有状态，需要断开后重建 erosaConnection.reconnect(); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 094d0b1d..6d69a836 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -2,24 +2,27 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.nio.charset.Charset; -import javax.annotation.Resource; - -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.filter.CanalEventFilter; import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; +import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.AbstractEventParser; import com.alibaba.otter.canal.parse.inbound.BinlogParser; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDBBuilder; +import com.alibaba.otter.canal.protocol.position.EntryPosition; public abstract class AbstractMysqlEventParser extends AbstractEventParser { protected final Logger logger = LoggerFactory.getLogger(this.getClass()); protected static final long BINLOG_START_OFFEST = 4L; + protected boolean enableTsdb = false; + protected String tsdbSpringXml; + protected TableMetaTSDB tableMetaTSDB; // 编码信息 protected byte connectionCharsetNumber = (byte) 33; protected Charset connectionCharset = Charset.forName("UTF-8"); @@ -28,9 +31,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { protected boolean filterQueryDdl = false; protected boolean filterRows = false; protected boolean filterTableError = false; - - @Resource - protected TableMetaManager tableMetaManager; protected boolean useDruidDdlFilter = true; protected BinlogParser buildParser() { @@ -49,9 +49,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { convert.setFilterQueryDdl(filterQueryDdl); convert.setFilterRows(filterRows); convert.setFilterTableError(filterTableError); - - // 初始化parser的时候也初始化管理mysql 表结构的管理器 - tableMetaManager.init(); return convert; } @@ -70,14 +67,34 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { * @param position * @return */ - protected boolean processTableMeta(BinlogPosition position) { - if (tableMetaManager != null) { - return tableMetaManager.rollback(position); + protected boolean processTableMeta(EntryPosition position) { + if (tableMetaTSDB != null) { + return tableMetaTSDB.rollback(position); } return true; } + public void start() throws CanalParseException { + if (enableTsdb) { + if (tableMetaTSDB == null) { + // 初始化 + tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + } + } + + super.start(); + } + + public void stop() throws CanalParseException { + if (enableTsdb) { + TableMetaTSDBBuilder.destory(destination); + tableMetaTSDB = null; + } + + super.stop(); + } + public void setEventBlackFilter(CanalEventFilter eventBlackFilter) { super.setEventBlackFilter(eventBlackFilter); @@ -122,14 +139,6 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { this.filterTableError = filterTableError; } - public TableMetaManager getTableMetaManager() { - return tableMetaManager; - } - - public void setTableMetaManager(TableMetaManager tableMetaManager) { - this.tableMetaManager = tableMetaManager; - } - public boolean isUseDruidDdlFilter() { return useDruidDdlFilter; } @@ -137,4 +146,24 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { public void setUseDruidDdlFilter(boolean useDruidDdlFilter) { this.useDruidDdlFilter = useDruidDdlFilter; } + + public void setEnableTsdb(boolean enableTsdb) { + this.enableTsdb = enableTsdb; + if (this.enableTsdb) { + if (tableMetaTSDB == null) { + // 初始化 + tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + } + } + } + + public void setTsdbSpringXml(String tsdbSpringXml) { + this.tsdbSpringXml = tsdbSpringXml; + + if (tableMetaTSDB == null) { + // 初始化 + tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + } + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java index d45c5615..d2c2fd2e 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java @@ -50,7 +50,7 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements throw new CanalParseException(e); } - tableMetaCache = new TableMetaCache(metaConnection, tableMetaManager); + tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 4a188b01..912787b4 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -26,6 +26,7 @@ import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogFormat; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogImage; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.position.EntryPosition; @@ -117,12 +118,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } } - if (tableMetaManager != null) { - tableMetaManager.setConnection(metaConnection); - tableMetaManager.setFilter(eventFilter); + if (tableMetaTSDB != null && tableMetaTSDB instanceof TableMetaManager) { + ((TableMetaManager) tableMetaTSDB).setConnection(metaConnection); + ((TableMetaManager) tableMetaTSDB).setFilter(eventFilter); + ((TableMetaManager) tableMetaTSDB).setBlackFilter(eventBlackFilter); } - tableMetaCache = new TableMetaCache(metaConnection, tableMetaManager); + tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } } @@ -337,6 +339,34 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE return endPosition; } + protected EntryPosition findEndPositionWithMasterIdAndTimestamp(MysqlConnection connection) { + MysqlConnection mysqlConnection = (MysqlConnection) connection; + final EntryPosition endPosition = findEndPosition(mysqlConnection); + if (tableMetaTSDB != null) { + long startTimestamp = System.currentTimeMillis(); + return findAsPerTimestampInSpecificLogFile(mysqlConnection, + startTimestamp, + endPosition, + endPosition.getJournalName()); + } else { + return endPosition; + } + } + + protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) { + MysqlConnection mysqlConnection = (MysqlConnection) connection; + if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) { + // 使用一个未来极大的时间，基于位点进行定位 + long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年 + return findAsPerTimestampInSpecificLogFile(mysqlConnection, + startTimestamp, + fixedPosition, + fixedPosition.getJournalName()); + } else { + return fixedPosition; + } + } + protected EntryPosition findStartPositionInternal(ErosaConnection connection) { MysqlConnection mysqlConnection = (MysqlConnection) connection; LogPosition logPosition = logPositionManager.getLatestIndexBy(destination); @@ -350,7 +380,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } if (entryPosition == null) { - entryPosition = findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费 + entryPosition = findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费 } // 判断一下是否需要按时间订阅 @@ -362,13 +392,15 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp()); } else { logger.warn("prepare to find start position just show master status"); - return findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费 + return findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费 } } else { if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) { // 如果指定binlogName + offest，直接返回 + entryPosition = findPositionWithMasterIdAndTimestamp(mysqlConnection, entryPosition); logger.warn("prepare to find start position {}:{}:{}", - new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(), "" }); + new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(), + entryPosition.getTimestamp() }); return entryPosition; } else { EntryPosition specificLogFilePosition = null; @@ -695,6 +727,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE String logfilename = entry.getHeader().getLogfileName(); Long logfileoffset = entry.getHeader().getLogfileOffset(); Long logposTimestamp = entry.getHeader().getExecuteTime(); + Long serverId = entry.getHeader().getServerId(); if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) { @@ -715,13 +748,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // position = current + // data.length，代表该事务的下一条offest，避免多余的事务重复 if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) { - entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp); + entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); logger.debug("set {} to be pending start position before finding another proper one...", entryPosition); logPosition.setPostion(entryPosition); } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) { // 当前事务开始位点 - entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp); + entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); logger.debug("set {} to be pending start position before finding another proper one...", entryPosition); logPosition.setPostion(entryPosition); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 4f0ada45..538f6b95 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -35,8 +35,8 @@ import com.alibaba.otter.canal.protocol.CanalEntry.RowData; import com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin; import com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd; import com.alibaba.otter.canal.protocol.CanalEntry.Type; +import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.google.protobuf.ByteString; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import com.taobao.tddl.dbsync.binlog.LogEvent; import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent; @@ -189,8 +189,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar if (!isSeek) { // 使用新的表结构元数据管理方式 - BinlogPosition position = createPosition(event.getHeader()); - tableMetaCache.apply(position, event.getDbName(), queryString); + EntryPosition position = createPosition(event.getHeader()); + tableMetaCache.apply(position, event.getDbName(), queryString, null); } Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type); @@ -386,7 +386,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar table.getDbName(), table.getTableName(), eventType); - BinlogPosition position = createPosition(event.getHeader()); + EntryPosition position = createPosition(event.getHeader()); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); rowChangeBuider.setTableId(event.getTableId()); rowChangeBuider.setIsDdl(false); @@ -444,8 +444,11 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } - private BinlogPosition createPosition(LogHeader logHeader) { - return new BinlogPosition(binlogFileName, logHeader.getLogPos(), logHeader.getServerId(), logHeader.getWhen()); // 记录到秒 + private EntryPosition createPosition(LogHeader logHeader) { + return new EntryPosition(binlogFileName, + logHeader.getLogPos(), + logHeader.getWhen() * 1000L, + logHeader.getServerId()); // 记录到秒 } private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, @@ -468,7 +471,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } } - BinlogPosition position = createPosition(event.getHeader()); + EntryPosition position = createPosition(event.getHeader()); if (!existRDSNoPrimaryKey) { // online ddl增加字段操作步骤： // 1. 新增一张临时表，将需要做ddl表的数据全量导入 @@ -716,7 +719,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return true; } - private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, BinlogPosition position) { + private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) { try { return tableMetaCache.getTableMeta(dbName, tbName, useCache, position); } catch (Exception e) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index 23229859..ff66c875 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -14,11 +14,11 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; +import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; /** * 处理table meta解析和缓存 @@ -37,15 +37,15 @@ public class TableMetaCache { private MysqlConnection connection; private boolean isOnRDS = false; - private TableMetaManager tableMetaManager; + private TableMetaTSDB tableMetaTSDB; // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表 private LoadingCache tableMetaDB; - public TableMetaCache(MysqlConnection con, TableMetaManager tableMetaManager){ + public TableMetaCache(MysqlConnection con, TableMetaTSDB tableMetaTSDB){ this.connection = con; - this.tableMetaManager = tableMetaManager; + this.tableMetaTSDB = tableMetaTSDB; // 如果持久存储的表结构为空，从db里面获取下 - if (tableMetaManager == null) { + if (tableMetaTSDB == null) { this.tableMetaDB = CacheBuilder.newBuilder().build(new CacheLoader() { @Override @@ -124,14 +124,14 @@ public class TableMetaCache { return tableMetaDB.getUnchecked(getFullName(schema, table)); } - public TableMeta getTableMeta(String schema, String table, BinlogPosition position) { + public TableMeta getTableMeta(String schema, String table, EntryPosition position) { return getTableMeta(schema, table, true, position); } - public TableMeta getTableMeta(String schema, String table, boolean useCache, BinlogPosition position) { + public TableMeta getTableMeta(String schema, String table, boolean useCache, EntryPosition position) { TableMeta tableMeta = null; - if (tableMetaManager != null) { - tableMeta = tableMetaManager.find(schema, table); + if (tableMetaTSDB != null) { + tableMeta = tableMetaTSDB.find(schema, table); if (tableMeta == null) { // 因为条件变化，可能第一次的tableMeta没取到，需要从db获取一次，并记录到snapshot中 String fullName = getFullName(schema, table); @@ -142,8 +142,8 @@ public class TableMetaCache { createDDL = packet.getFieldValues().get(1); } // 强制覆盖掉内存值 - tableMetaManager.apply(position, schema, createDDL); - tableMeta = tableMetaManager.find(schema, table); + tableMetaTSDB.apply(position, schema, createDDL, "first"); + tableMeta = tableMetaTSDB.find(schema, table); } catch (IOException e) { throw new CanalParseException("fetch failed by table meta:" + fullName, e); } @@ -159,7 +159,7 @@ public class TableMetaCache { } public void clearTableMeta(String schema, String table) { - if (tableMetaManager != null) { + if (tableMetaTSDB != null) { // tsdb不需要做,会基于ddl sql自动清理 } else { tableMetaDB.invalidate(getFullName(schema, table)); @@ -167,7 +167,7 @@ public class TableMetaCache { } public void clearTableMetaWithSchemaName(String schema) { - if (tableMetaManager != null) { + if (tableMetaTSDB != null) { // tsdb不需要做,会基于ddl sql自动清理 } else { for (String name : tableMetaDB.asMap().keySet()) { @@ -180,7 +180,7 @@ public class TableMetaCache { } public void clearTableMeta() { - if (tableMetaManager != null) { + if (tableMetaTSDB != null) { // tsdb不需要做,会基于ddl sql自动清理 } else { tableMetaDB.invalidateAll(); @@ -195,9 +195,9 @@ public class TableMetaCache { * @param ddl * @return */ - public boolean apply(BinlogPosition position, String schema, String ddl) { - if (tableMetaManager != null) { - return tableMetaManager.apply(position, schema, ddl); + public boolean apply(EntryPosition position, String schema, String ddl, String extra) { + if (tableMetaTSDB != null) { + return tableMetaTSDB.apply(position, schema, ddl, extra); } else { // ignore return true; diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java deleted file mode 100644 index 25f60ee3..00000000 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DataSourceFactoryTSDB.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; - -import javax.sql.DataSource; - -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.vendor.MySqlExceptionSorter; -import com.alibaba.druid.pool.vendor.MySqlValidConnectionChecker; -import com.alibaba.otter.canal.parse.exception.CanalParseException; - -/** - * Created by wanshao Date: 2017/9/22 Time: 下午2:46 - **/ -public class DataSourceFactoryTSDB { - - public static DataSource getDataSource(String url, String userName, String password) { - try { - DruidDataSource druidDataSource = new DruidDataSource(); - druidDataSource.setUrl(url); - druidDataSource.setUsername(userName); - druidDataSource.setPassword(password); - druidDataSource.setTestWhileIdle(true); - druidDataSource.setTestOnBorrow(false); - druidDataSource.setTestOnReturn(false); - druidDataSource.setNotFullTimeoutRetryCount(2); - druidDataSource.setValidConnectionCheckerClassName(MySqlValidConnectionChecker.class.getName()); - druidDataSource.setExceptionSorterClassName(MySqlExceptionSorter.class.getName()); - druidDataSource.setValidationQuery("SELECT 1"); - druidDataSource.setInitialSize(1); - druidDataSource.setMinIdle(1); - druidDataSource.setMaxActive(30); - druidDataSource.setMaxWait(10 * 1000); - druidDataSource.setTimeBetweenEvictionRunsMillis(60 * 1000); - druidDataSource.setMinEvictableIdleTimeMillis(50 * 1000); - druidDataSource.setUseUnfairLock(true); - druidDataSource.init(); - return druidDataSource; - } catch (Throwable e) { - throw new CanalParseException("init druidDataSource failed", e); - } - } -} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java index f3c6aeab..3bd220ef 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -6,14 +6,15 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.alibaba.druid.sql.ast.SQLDataType; +import com.alibaba.druid.sql.ast.SQLDataTypeImpl; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLCharExpr; import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; import com.alibaba.druid.sql.ast.expr.SQLNullExpr; import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; @@ -30,11 +31,10 @@ import com.alibaba.druid.sql.repository.Schema; import com.alibaba.druid.sql.repository.SchemaObject; import com.alibaba.druid.sql.repository.SchemaRepository; import com.alibaba.druid.util.JdbcConstants; - import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; +import com.alibaba.otter.canal.protocol.position.EntryPosition; /** * 基于DDL维护的内存表结构 @@ -44,15 +44,19 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; */ public class MemoryTableMeta implements TableMetaTSDB { + private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class); private Map, TableMeta> tableMetas = new ConcurrentHashMap, TableMeta>(); private SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL); - private Logger logger = LoggerFactory.getLogger(MemoryTableMeta.class); - public MemoryTableMeta(Logger logger){ - this.logger = logger; + public MemoryTableMeta(){ } - public boolean apply(BinlogPosition position, String schema, String ddl) { + @Override + public boolean init(String destination) { + return true; + } + + public boolean apply(EntryPosition position, String schema, String ddl, String extra) { tableMetas.clear(); synchronized (this) { if (StringUtils.isNotEmpty(schema)) { @@ -116,7 +120,7 @@ public class MemoryTableMeta implements TableMetaTSDB { } @Override - public boolean rollback(BinlogPosition position) { + public boolean rollback(EntryPosition position) { throw new RuntimeException("not support for memory"); } @@ -169,6 +173,17 @@ public class MemoryTableMeta implements TableMetaTSDB { dataTypStr += ")"; } + if (dataType instanceof SQLDataTypeImpl) { + SQLDataTypeImpl dataTypeImpl = (SQLDataTypeImpl) dataType; + if (dataTypeImpl.isUnsigned()) { + dataTypStr += " unsigned"; + } + + if (dataTypeImpl.isZerofill()) { + dataTypStr += " zerofill"; + } + } + if (column.getDefaultExpr() == null || column.getDefaultExpr() instanceof SQLNullExpr) { fieldMeta.setDefaultValue(null); } else { @@ -211,6 +226,8 @@ public class MemoryTableMeta implements TableMetaTSDB { + DruidDdlParser.unescapeName(((SQLPropertyExpr) sqlName).getName()); } else if (sqlName instanceof SQLIdentifierExpr) { return DruidDdlParser.unescapeName(((SQLIdentifierExpr) sqlName).getName()); + } else if (sqlName instanceof SQLCharExpr) { + return ((SQLCharExpr) sqlName).getText(); } else { return sqlName.toString(); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java index 9d926548..c8fe89e1 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java @@ -11,13 +11,12 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; -import javax.annotation.Resource; - import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import com.alibaba.druid.sql.repository.Schema; import com.alibaba.fastjson.JSON; @@ -32,10 +31,10 @@ import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DdlResult; import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDAO; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO; +import com.alibaba.otter.canal.protocol.position.EntryPosition; /** * 基于console远程管理 see internal class: CanalTableMeta , ConsoleTableMetaTSDB @@ -45,23 +44,27 @@ import com.taobao.tddl.dbsync.binlog.BinlogPosition; */ public class TableMetaManager implements TableMetaTSDB { - private static Logger logger = LoggerFactory.getLogger(TableMetaManager.class); - private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); - private static final BinlogPosition INIT_POSITION = BinlogPosition.parseFromString("0:0#-2.-1"); - private MemoryTableMeta memoryTableMeta; - private MysqlConnection connection; // 查询meta信息的链接 - private CanalEventFilter filter; - private BinlogPosition lastPosition; - private ScheduledExecutorService scheduler; + private static Logger logger = LoggerFactory.getLogger(TableMetaManager.class); + private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); + private static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L); + private String destination; + private MemoryTableMeta memoryTableMeta; + private MysqlConnection connection; // 查询meta信息的链接 + private CanalEventFilter filter; + private CanalEventFilter blackFilter; + private EntryPosition lastPosition; + private ScheduledExecutorService scheduler; + private MetaHistoryDAO metaHistoryDAO; + private MetaSnapshotDAO metaSnapshotDAO; - @Resource - private MetaHistoryDAO metaHistoryDAO; + public TableMetaManager(){ - @Resource - private MetaSnapshotDAO metaSnapshotDAO; + } - public void init() { - this.memoryTableMeta = new MemoryTableMeta(logger); + @Override + public boolean init(final String destination) { + this.destination = destination; + this.memoryTableMeta = new MemoryTableMeta(); this.scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { @Override @@ -76,17 +79,14 @@ public class TableMetaManager implements TableMetaTSDB { @Override public void run() { try { - logger.info("-------- begin to produce snapshot for table meta"); + MDC.put("destination", destination); applySnapshotToDB(lastPosition, false); } catch (Throwable e) { - logger.error("scheudle faield", e); + logger.error("scheudle applySnapshotToDB faield", e); } } - }, 24, 24, TimeUnit.SECONDS); - } - - public TableMetaManager(){ - + }, 24, 24, TimeUnit.HOURS); + return true; } @Override @@ -97,13 +97,13 @@ public class TableMetaManager implements TableMetaTSDB { } @Override - public boolean apply(BinlogPosition position, String schema, String ddl) { + public boolean apply(EntryPosition position, String schema, String ddl, String extra) { // 首先记录到内存结构 synchronized (memoryTableMeta) { - if (memoryTableMeta.apply(position, schema, ddl)) { + if (memoryTableMeta.apply(position, schema, ddl, extra)) { this.lastPosition = position; // 同步每次变更给远程做历史记录 - return applyHistoryToDB(position, schema, ddl); + return applyHistoryToDB(position, schema, ddl, extra); } else { throw new RuntimeException("apply to memory is failed"); } @@ -111,11 +111,11 @@ public class TableMetaManager implements TableMetaTSDB { } @Override - public boolean rollback(BinlogPosition position) { + public boolean rollback(EntryPosition position) { // 每次rollback需要重新构建一次memory data - this.memoryTableMeta = new MemoryTableMeta(logger); + this.memoryTableMeta = new MemoryTableMeta(); boolean flag = false; - BinlogPosition snapshotPosition = buildMemFromSnapshot(position); + EntryPosition snapshotPosition = buildMemFromSnapshot(position); if (snapshotPosition != null) { applyHistoryOnMemory(snapshotPosition, position); flag = true; @@ -145,20 +145,25 @@ public class TableMetaManager implements TableMetaTSDB { ResultSetPacket packet = connection.query("show databases"); List schemas = new ArrayList(); for (String schema : packet.getFieldValues()) { - if (!filter.filter(schema)) { - schemas.add(schema); - } + schemas.add(schema); } for (String schema : schemas) { packet = connection.query("show tables from `" + schema + "`"); List tables = new ArrayList(); for (String table : packet.getFieldValues()) { - if (!filter.filter(table)) { - tables.add(table); + String fullName = schema + "." + table; + if (blackFilter == null || !blackFilter.filter(fullName)) { + if (filter == null || filter.filter(fullName)) { + tables.add(table); + } } } + if (tables.isEmpty()) { + continue; + } + StringBuilder sql = new StringBuilder(); for (String table : tables) { sql.append("show create table `" + schema + "`.`" + table + "`;"); @@ -168,7 +173,7 @@ public class TableMetaManager implements TableMetaTSDB { for (ResultSetPacket onePacket : packets) { if (onePacket.getFieldValues().size() > 1) { String oneTableCreateSql = onePacket.getFieldValues().get(1); - memoryTableMeta.apply(INIT_POSITION, schema, oneTableCreateSql); + memoryTableMeta.apply(INIT_POSITION, schema, oneTableCreateSql, null); } } } @@ -179,25 +184,26 @@ public class TableMetaManager implements TableMetaTSDB { } } - private boolean applyHistoryToDB(BinlogPosition position, String schema, String ddl) { + private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) { Map content = new HashMap(); - content.put("binlogFile", position.getFileName()); + content.put("destination", destination); + content.put("binlogFile", position.getJournalName()); content.put("binlogOffest", String.valueOf(position.getPosition())); - content.put("binlogMasterId", String.valueOf(position.getMasterId())); + content.put("binlogMasterId", String.valueOf(position.getServerId())); content.put("binlogTimestamp", String.valueOf(position.getTimestamp())); content.put("useSchema", schema); if (content.isEmpty()) { throw new RuntimeException("apply failed caused by content is empty in applyHistoryToDB"); } // 待补充 - List ddlResults = DruidDdlParser.parse(schema, ddl); + List ddlResults = DruidDdlParser.parse(ddl, schema); if (ddlResults.size() > 0) { DdlResult ddlResult = ddlResults.get(0); - content.put("schema", ddlResult.getSchemaName()); - content.put("table", ddlResult.getTableName()); - content.put("type", ddlResult.getType().name()); - content.put("sql", ddl); - // content.put("extra", ""); + content.put("sqlSchema", ddlResult.getSchemaName()); + content.put("sqlTable", ddlResult.getTableName()); + content.put("sqlType", ddlResult.getType().name()); + content.put("sqlText", ddl); + content.put("extra", extra); } MetaHistoryDO metaDO = new MetaHistoryDO(); @@ -212,7 +218,7 @@ public class TableMetaManager implements TableMetaTSDB { // 忽略掉重复的位点 logger.warn("dup apply for sql : " + ddl); } else { - throw new CanalParseException("apply history to db failed caused by : " + e.getMessage()); + throw new CanalParseException("apply history to db failed caused by : " + e.getMessage(), e); } } @@ -222,9 +228,9 @@ public class TableMetaManager implements TableMetaTSDB { /** * 发布数据到console上 */ - private boolean applySnapshotToDB(BinlogPosition position, boolean init) { + private boolean applySnapshotToDB(EntryPosition position, boolean init) { // 获取一份快照 - MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta(logger); + MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta(); Map schemaDdls = null; synchronized (memoryTableMeta) { if (!init && position == null) { @@ -233,7 +239,7 @@ public class TableMetaManager implements TableMetaTSDB { } schemaDdls = memoryTableMeta.snapshot(); for (Map.Entry entry : schemaDdls.entrySet()) { - tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue()); + tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue(), null); } } @@ -248,9 +254,10 @@ public class TableMetaManager implements TableMetaTSDB { } if (compareAll) { Map content = new HashMap(); - content.put("binlogFile", position.getFileName()); + content.put("destination", destination); + content.put("binlogFile", position.getJournalName()); content.put("binlogOffest", String.valueOf(position.getPosition())); - content.put("binlogMasterId", String.valueOf(position.getMasterId())); + content.put("binlogMasterId", String.valueOf(position.getServerId())); content.put("binlogTimestamp", String.valueOf(position.getTimestamp())); content.put("data", JSON.toJSONString(schemaDdls)); if (content.isEmpty()) { @@ -264,9 +271,9 @@ public class TableMetaManager implements TableMetaTSDB { } catch (Throwable e) { if (isUkDuplicateException(e)) { // 忽略掉重复的位点 - logger.warn("dup apply snapshot for data : " + snapshotDO.getData()); + logger.info("dup apply snapshot use position : " + position + " , just ignore"); } else { - throw new CanalParseException("apply failed caused by : " + e.getMessage()); + throw new CanalParseException("apply failed caused by : " + e.getMessage(), e); } } return true; @@ -296,18 +303,21 @@ public class TableMetaManager implements TableMetaTSDB { return result; } - private BinlogPosition buildMemFromSnapshot(BinlogPosition position) { + private EntryPosition buildMemFromSnapshot(EntryPosition position) { try { - MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(position.getTimestamp()); + MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp()); + if (snapshotDO == null) { + return null; + } String binlogFile = snapshotDO.getBinlogFile(); Long binlogOffest = snapshotDO.getBinlogOffest(); String binlogMasterId = snapshotDO.getBinlogMasterId(); Long binlogTimestamp = snapshotDO.getBinlogTimestamp(); - BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, + EntryPosition snapshotPosition = new EntryPosition(binlogFile, binlogOffest == null ? 0l : binlogOffest, - Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), - binlogTimestamp == null ? 0l : binlogTimestamp); + binlogTimestamp == null ? 0l : binlogTimestamp, + Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId)); // data存储为Map，每个分库一套建表 String sqlData = snapshotDO.getData(); JSONObject jsonObj = JSON.parseObject(sqlData); @@ -315,43 +325,49 @@ public class TableMetaManager implements TableMetaTSDB { // 记录到内存 if (!memoryTableMeta.apply(snapshotPosition, ObjectUtils.toString(entry.getKey()), - ObjectUtils.toString(entry.getValue()))) { + ObjectUtils.toString(entry.getValue()), + null)) { return null; } } return snapshotPosition; } catch (Throwable e) { - throw new CanalParseException("apply failed caused by : " + e.getMessage()); + throw new CanalParseException("apply failed caused by : " + e.getMessage(), e); } } - private boolean applyHistoryOnMemory(BinlogPosition position, BinlogPosition rollbackPosition) { + private boolean applyHistoryOnMemory(EntryPosition position, EntryPosition rollbackPosition) { try { - List metaHistoryDOList = metaHistoryDAO.findByTimestamp(position.getTimestamp(), + List metaHistoryDOList = metaHistoryDAO.findByTimestamp(destination, + position.getTimestamp(), rollbackPosition.getTimestamp()); + if (metaHistoryDOList == null) { + return true; + } + for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) { String binlogFile = metaHistoryDO.getBinlogFile(); Long binlogOffest = metaHistoryDO.getBinlogOffest(); String binlogMasterId = metaHistoryDO.getBinlogMasterId(); Long binlogTimestamp = metaHistoryDO.getBinlogTimestamp(); String useSchema = metaHistoryDO.getUseSchema(); - String sqlData = metaHistoryDO.getSql(); - BinlogPosition snapshotPosition = new BinlogPosition(binlogFile, + String sqlData = metaHistoryDO.getSqlText(); + EntryPosition snapshotPosition = new EntryPosition(binlogFile, binlogOffest == null ? 0L : binlogOffest, - Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId), - binlogTimestamp == null ? 0L : binlogTimestamp); + binlogTimestamp == null ? 0L : binlogTimestamp, + Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId)); // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之 if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) { continue; - } else if (rollbackPosition.getMasterId() == snapshotPosition.getMasterId() + } else if (rollbackPosition.getServerId() == snapshotPosition.getServerId() && snapshotPosition.compareTo(rollbackPosition) > 0) { continue; } // 记录到内存 - if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData)) { + if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData, null)) { return false; } @@ -441,6 +457,10 @@ public class TableMetaManager implements TableMetaTSDB { this.metaSnapshotDAO = metaSnapshotDAO; } + public void setBlackFilter(CanalEventFilter blackFilter) { + this.blackFilter = blackFilter; + } + public MysqlConnection getConnection() { return connection; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java index 44cd9959..d712adbe 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDB.java @@ -3,17 +3,21 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; import java.util.Map; import com.alibaba.otter.canal.parse.inbound.TableMeta; - -import com.taobao.tddl.dbsync.binlog.BinlogPosition; +import com.alibaba.otter.canal.protocol.position.EntryPosition; /** * 表结构的时间序列存储 * * @author agapple 2017年7月27日 下午4:06:30 - * @since 3.2.5 + * @since 1.0.25 */ public interface TableMetaTSDB { + /** + * 初始化 + */ + public boolean init(String destination); + /** * 获取当前的表结构 */ @@ -22,12 +26,12 @@ public interface TableMetaTSDB { /** * 添加ddl到时间序列库中 */ - public boolean apply(BinlogPosition position, String schema, String ddl); + public boolean apply(EntryPosition position, String schema, String ddl, String extra); /** * 回滚到指定位点的表结构 */ - public boolean rollback(BinlogPosition position); + public boolean rollback(EntryPosition position); /** * 生成快照内容 diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java new file mode 100644 index 00000000..8e37ef71 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaTSDBBuilder.java @@ -0,0 +1,51 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import java.util.concurrent.ConcurrentMap; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import com.google.common.collect.Maps; + +/** + * @author agapple 2017年10月11日 下午8:45:40 + * @since 1.0.25 + */ +public class TableMetaTSDBBuilder { + + protected final static Logger logger = LoggerFactory.getLogger(TableMetaTSDBBuilder.class); + private static ConcurrentMap contexts = Maps.newConcurrentMap(); + + /** + * 代理一下tableMetaTSDB的获取,使用隔离的spring定义 + */ + public static TableMetaTSDB build(String destination, String springXml) { + if (StringUtils.isNotEmpty(springXml)) { + ClassPathXmlApplicationContext applicationContext = contexts.get(destination); + if (applicationContext == null) { + synchronized (contexts) { + if (applicationContext == null) { + applicationContext = new ClassPathXmlApplicationContext(springXml); + contexts.put(destination, applicationContext); + } + } + } + TableMetaTSDB tableMetaTSDB = (TableMetaTSDB) applicationContext.getBean("tableMetaTSDB"); + tableMetaTSDB.init(destination); + logger.info("{} init TableMetaTSDB with {}", destination, springXml); + return tableMetaTSDB; + } else { + return null; + } + } + + public static void destory(String destination) { + ClassPathXmlApplicationContext context = contexts.remove(destination); + if (context != null) { + logger.info("{} destory TableMetaTSDB", destination); + context.close(); + } + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java index 46ffafbf..e43c5e62 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java @@ -1,11 +1,19 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.HashMap; import java.util.List; +import javax.sql.DataSource; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; import com.google.common.collect.Maps; /** @@ -14,22 +22,25 @@ import com.google.common.collect.Maps; * @author wanshao 2017年7月27日 下午10:51:55 * @since 3.2.5 */ +@SuppressWarnings("deprecation") public class MetaHistoryDAO extends SqlMapClientDaoSupport { - public List getAll() { - return getSqlMapClientTemplate().queryForList("table_meta_history.getAll"); - } - public Long insert(MetaHistoryDO metaDO) { - return (Long) getSqlMapClientTemplate().insert("table_meta_history.insert", metaDO); + return (Long) getSqlMapClientTemplate().insert("meta_history.insert", metaDO); } - public List findByTimestamp(long snapshotTimestamp, long timestamp) { + public List findByTimestamp(String destination, Long snapshotTimestamp, Long timestamp) { HashMap params = Maps.newHashMapWithExpectedSize(2); - params.put("snapshotTimestamp", snapshotTimestamp); - params.put("timestamp", timestamp); - return (List) getSqlMapClientTemplate().queryForList("table_meta_history.findByTimestamp", - params); + params.put("destination", destination); + params.put("snapshotTimestamp", snapshotTimestamp == null ? 0L : snapshotTimestamp); + params.put("timestamp", timestamp == null ? 0L : timestamp); + return (List) getSqlMapClientTemplate().queryForList("meta_history.findByTimestamp", params); + } + + public Integer deleteByName(String destination) { + HashMap params = Maps.newHashMapWithExpectedSize(2); + params.put("destination", destination); + return getSqlMapClientTemplate().delete("meta_history.deleteByName", params); } /** @@ -37,7 +48,35 @@ public class MetaHistoryDAO extends SqlMapClientDaoSupport { */ public Integer deleteByGmtModified(int interval) { HashMap params = Maps.newHashMapWithExpectedSize(2); - params.put("interval", interval); - return getSqlMapClientTemplate().delete("table_meta_history.deleteByGmtModified", params); + long timestamp = System.currentTimeMillis() - interval * 1000; + Date date = new Date(timestamp); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + params.put("timestamp", format.format(date)); + return getSqlMapClientTemplate().delete("meta_history.deleteByGmtModified", params); + } + + protected void initDao() throws Exception { + Connection conn = null; + InputStream input = null; + try { + DataSource dataSource = getDataSource(); + conn = dataSource.getConnection(); + input = Thread.currentThread().getContextClassLoader().getResourceAsStream("ddl/mysql/meta_history.sql"); + if (input == null) { + return; + } + + String sql = StringUtils.join(IOUtils.readLines(input), "\n"); + Statement stmt = conn.createStatement(); + stmt.execute(sql); + stmt.close(); + } catch (Throwable e) { + logger.warn("init meta_history failed", e); + } finally { + IOUtils.closeQuietly(input); + if (conn != null) { + conn.close(); + } + } } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDO.java similarity index 55% rename from parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java rename to parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDO.java index 96108515..6ac548f1 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaHistoryDO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDO.java @@ -1,37 +1,26 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model; +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; import java.util.Date; /** * @author agapple 2017年7月27日 下午11:09:41 - * @since 3.2.5 + * @since 1.0.25 */ public class MetaHistoryDO { - /** - * 主键 - */ private Long id; - - /** - * 创建时间 - */ private Date gmtCreate; - - /** - * 修改时间 - */ private Date gmtModified; - + private String destination; private String binlogFile; private Long binlogOffest; private String binlogMasterId; private Long binlogTimestamp; private String useSchema; - private String schema; - private String table; - private String sql; - private String type; + private String sqlSchema; + private String sqlTable; + private String sqlText; + private String sqlType; private String extra; public Long getId() { @@ -98,37 +87,6 @@ public class MetaHistoryDO { this.useSchema = useSchema; } - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } public String getExtra() { return extra; @@ -138,4 +96,53 @@ public class MetaHistoryDO { this.extra = extra; } + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public String getSqlSchema() { + return sqlSchema; + } + + public void setSqlSchema(String sqlSchema) { + this.sqlSchema = sqlSchema; + } + + public String getSqlTable() { + return sqlTable; + } + + public void setSqlTable(String sqlTable) { + this.sqlTable = sqlTable; + } + + public String getSqlText() { + return sqlText; + } + + public void setSqlText(String sqlText) { + this.sqlText = sqlText; + } + + public String getSqlType() { + return sqlType; + } + + public void setSqlType(String sqlType) { + this.sqlType = sqlType; + } + + @Override + public String toString() { + return "MetaHistoryDO [id=" + id + ", gmtCreate=" + gmtCreate + ", gmtModified=" + gmtModified + + ", destination=" + destination + ", binlogFile=" + binlogFile + ", binlogOffest=" + binlogOffest + + ", binlogMasterId=" + binlogMasterId + ", binlogTimestamp=" + binlogTimestamp + ", useSchema=" + + useSchema + ", sqlSchema=" + sqlSchema + ", sqlTable=" + sqlTable + ", sqlText=" + sqlText + + ", sqlType=" + sqlType + ", extra=" + extra + "]"; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java index ca32ebd3..1a3f3053 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java @@ -1,10 +1,18 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.Statement; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.HashMap; +import javax.sql.DataSource; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaSnapshotDO; import com.google.common.collect.Maps; /** @@ -13,27 +21,29 @@ import com.google.common.collect.Maps; * @author wanshao 2017年7月27日 下午10:51:55 * @since 3.2.5 */ - +@SuppressWarnings("deprecation") public class MetaSnapshotDAO extends SqlMapClientDaoSupport { public Long insert(MetaSnapshotDO snapshotDO) { - return (Long) getSqlMapClientTemplate().insert("table_meta_snapshot.insert", snapshotDO); + return (Long) getSqlMapClientTemplate().insert("meta_snapshot.insert", snapshotDO); } public Long update(MetaSnapshotDO snapshotDO) { - return (Long) getSqlMapClientTemplate().insert("table_meta_snapshot.update", snapshotDO); + return (Long) getSqlMapClientTemplate().insert("meta_snapshot.update", snapshotDO); } - public MetaSnapshotDO findByTimestamp(long timestamp) { + public MetaSnapshotDO findByTimestamp(String destination, Long timestamp) { HashMap params = Maps.newHashMapWithExpectedSize(2); - params.put("timestamp", timestamp); - return (MetaSnapshotDO) getSqlMapClientTemplate().queryForObject("table_meta_snapshot.findByTimestamp", params); + params.put("timestamp", timestamp == null ? 0L : timestamp); + params.put("destination", destination); + + return (MetaSnapshotDO) getSqlMapClientTemplate().queryForObject("meta_snapshot.findByTimestamp", params); } - public Integer deleteByTask(String taskName) { + public Integer deleteByName(String destination) { HashMap params = Maps.newHashMapWithExpectedSize(2); - params.put("taskName", taskName); - return getSqlMapClientTemplate().delete("table_meta_snapshot.deleteByTaskName", params); + params.put("destination", destination); + return getSqlMapClientTemplate().delete("meta_snapshot.deleteByName", params); } /** @@ -41,8 +51,35 @@ public class MetaSnapshotDAO extends SqlMapClientDaoSupport { */ public Integer deleteByGmtModified(int interval) { HashMap params = Maps.newHashMapWithExpectedSize(2); - params.put("interval", interval); - return getSqlMapClientTemplate().delete("table_meta_snapshot.deleteByGmtModified", params); + long timestamp = System.currentTimeMillis() - interval * 1000; + Date date = new Date(timestamp); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + params.put("timestamp", format.format(date)); + return getSqlMapClientTemplate().delete("meta_snapshot.deleteByGmtModified", params); } + protected void initDao() throws Exception { + Connection conn = null; + InputStream input = null; + try { + DataSource dataSource = getDataSource(); + conn = dataSource.getConnection(); + input = Thread.currentThread().getContextClassLoader().getResourceAsStream("ddl/mysql/meta_snapshot.sql"); + if (input == null) { + return; + } + + String sql = StringUtils.join(IOUtils.readLines(input), "\n"); + Statement stmt = conn.createStatement(); + stmt.execute(sql); + stmt.close(); + } catch (Throwable e) { + logger.warn("init meta_history failed", e); + } finally { + IOUtils.closeQuietly(input); + if (conn != null) { + conn.close(); + } + } + } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDO.java similarity index 72% rename from parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java rename to parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDO.java index 8ddfbc64..693c5f8e 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/model/MetaSnapshotDO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDO.java @@ -1,28 +1,17 @@ -package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model; +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; import java.util.Date; /** * @author agapple 2017年7月27日 下午11:09:41 - * @since 3.2.5 + * @since 1.0.25 */ public class MetaSnapshotDO { - /** - * 主键 - */ private Long id; - - /** - * 创建时间 - */ private Date gmtCreate; - - /** - * 修改时间 - */ private Date gmtModified; - + private String destination; private String binlogFile; private Long binlogOffest; private String binlogMasterId; @@ -102,4 +91,20 @@ public class MetaSnapshotDO { this.extra = extra; } + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + @Override + public String toString() { + return "MetaSnapshotDO [id=" + id + ", gmtCreate=" + gmtCreate + ", gmtModified=" + gmtModified + + ", destination=" + destination + ", binlogFile=" + binlogFile + ", binlogOffest=" + binlogOffest + + ", binlogMasterId=" + binlogMasterId + ", binlogTimestamp=" + binlogTimestamp + ", data=" + data + + ", extra=" + extra + "]"; + } + } diff --git a/parse/src/main/resources/ddl/derby/meta_history.sql b/parse/src/main/resources/ddl/derby/meta_history.sql new file mode 100644 index 00000000..0529e1fa --- /dev/null +++ b/parse/src/main/resources/ddl/derby/meta_history.sql @@ -0,0 +1,22 @@ +CREATE TABLE meta_history ( + id bigint GENERATED ALWAYS AS IDENTITY NOT NULL, + gmt_create timestamp NOT NULL, + gmt_modified timestamp NOT NULL, + destination varchar(128) DEFAULT NULL, + binlog_file varchar(64) DEFAULT NULL, + binlog_offest bigint DEFAULT NULL, + binlog_master_id varchar(64) DEFAULT NULL, + binlog_timestamp bigint DEFAULT NULL, + use_schema varchar(1024) DEFAULT NULL, + sql_schema varchar(1024) DEFAULT NULL, + sql_table varchar(1024) DEFAULT NULL, + sql_text clob(16 M) DEFAULT NULL, + sql_type varchar(1024) DEFAULT NULL, + extra varchar(512) DEFAULT NULL, + PRIMARY KEY (id), + CONSTRAINT meta_history_binlog_file_offest UNIQUE (destination,binlog_master_id,binlog_file,binlog_offest) +); + +create index meta_history_destination on meta_history(destination); +create index meta_history_destination_timestamp on meta_history(destination,binlog_timestamp); +create index meta_history_gmt_modified on meta_history(gmt_modified); \ No newline at end of file diff --git a/parse/src/main/resources/ddl/derby/meta_snapshot.sql b/parse/src/main/resources/ddl/derby/meta_snapshot.sql new file mode 100644 index 00000000..61e6cd53 --- /dev/null +++ b/parse/src/main/resources/ddl/derby/meta_snapshot.sql @@ -0,0 +1,18 @@ +CREATE TABLE meta_snapshot ( + id bigint GENERATED ALWAYS AS IDENTITY NOT NULL, + gmt_create timestamp NOT NULL, + gmt_modified timestamp NOT NULL, + destination varchar(128) DEFAULT NULL, + binlog_file varchar(64) DEFAULT NULL, + binlog_offest bigint DEFAULT NULL, + binlog_master_id varchar(64) DEFAULT NULL, + binlog_timestamp bigint DEFAULT NULL, + data clob(16 M) DEFAULT NULL, + extra varchar(512) DEFAULT NULL, + PRIMARY KEY (id), + CONSTRAINT meta_snapshot_binlog_file_offest UNIQUE (destination,binlog_master_id,binlog_file,binlog_offest) +); + +create index meta_snapshot_destination on meta_snapshot(destination); +create index meta_snapshot_destination_timestamp on meta_snapshot(destination,binlog_timestamp); +create index meta_snapshot_gmt_modified on meta_snapshot(gmt_modified); \ No newline at end of file diff --git a/parse/src/main/resources/ddl/mysql/meta_history.sql b/parse/src/main/resources/ddl/mysql/meta_history.sql new file mode 100644 index 00000000..36425961 --- /dev/null +++ b/parse/src/main/resources/ddl/mysql/meta_history.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS `meta_history` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', + `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', + `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table', + `sql_text` longtext DEFAULT NULL COMMENT '执行的sql', + `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构变化明细表'; \ No newline at end of file diff --git a/parse/src/main/resources/ddl/mysql/meta_snapshot.sql b/parse/src/main/resources/ddl/mysql/meta_snapshot.sql new file mode 100644 index 00000000..43313cb6 --- /dev/null +++ b/parse/src/main/resources/ddl/mysql/meta_snapshot.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS `meta_snapshot` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `data` longtext DEFAULT NULL COMMENT '表结构数据', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `destination` (`destination`), + KEY `destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `gmt_modified` (`gmt_modified`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='表结构记录表快照表'; \ No newline at end of file diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java index 7b996ba2..ba8854a7 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java @@ -31,7 +31,7 @@ public class LocalBinlogEventParserTest { public void setUp() { URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); File dummyFile = new File(url.getFile()); - directory = new File("/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb").getPath(); + directory = new File(dummyFile + "/binlog/tsdb").getPath(); } @Test diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java index d8915e64..aef55822 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java @@ -7,6 +7,7 @@ import java.util.List; import org.junit.Assert; import org.junit.Test; +import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter; import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager; import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest; @@ -26,13 +27,18 @@ public class MysqlDumpTest { @Test public void testSimple() { final MysqlEventParser controller = new MysqlEventParser(); - final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L); + final EntryPosition startPosition = new EntryPosition("mysql-bin.000001", 104606L); controller.setConnectionCharset(Charset.forName("UTF-8")); controller.setSlaveId(3344L); controller.setDetectingEnable(false); controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal")); controller.setMasterPosition(startPosition); + controller.setEnableTsdb(true); + controller.setDestination("example"); + controller.setTsdbSpringXml("classpath:tsdb/mysql-tsdb.xml"); + controller.setEventFilter(new AviaterRegexFilter("test\\..*")); + controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*")); controller.setEventSink(new AbstractCanalEventSinkTest>() { public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination) @@ -100,7 +106,7 @@ public class MysqlDumpTest { controller.start(); try { - Thread.sleep(100 * 1000L); + Thread.sleep(100 * 1000 * 1000L); } catch (InterruptedException e) { Assert.fail(e.getMessage()); } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java index 8ead5e2a..9ef0dfc3 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMetaTest.java @@ -4,32 +4,31 @@ import java.io.File; import java.io.FileInputStream; import java.net.URL; -import com.alibaba.otter.canal.parse.inbound.TableMeta; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; - import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import com.alibaba.otter.canal.parse.inbound.TableMeta; + /** * @author agapple 2017年8月1日 下午7:15:54 */ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = { "/dal-dao.xml" }) +@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" }) public class MemoryTableMetaTest { @Test public void testSimple() throws Throwable { - MemoryTableMeta memoryTableMeta = new MemoryTableMeta(null); + MemoryTableMeta memoryTableMeta = new MemoryTableMeta(); URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); File dummyFile = new File(url.getFile()); File create = new File(dummyFile.getParent() + "/ddl", "create.sql"); String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); - memoryTableMeta.apply(null, "test", sql); + memoryTableMeta.apply(null, "test", sql, null); TableMeta meta = memoryTableMeta.find("test", "test"); System.out.println(meta); diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java index 29c02f7e..b199a326 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MetaHistoryDAOTest.java @@ -4,27 +4,27 @@ import java.util.List; import javax.annotation.Resource; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.model.MetaHistoryDO; - import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDAO; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaHistoryDO; + /** * Created by wanshao Date: 2017/9/20 Time: 下午5:00 **/ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = { "/dal-dao.xml" }) +@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" }) public class MetaHistoryDAOTest { @Resource MetaHistoryDAO metaHistoryDAO; @Test - public void testGetAll() { - List metaHistoryDOList = metaHistoryDAO.getAll(); + public void testSimple() { + List metaHistoryDOList = metaHistoryDAO.findByTimestamp("test", 0L, 0L); for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) { System.out.println(metaHistoryDO.getId()); } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java new file mode 100644 index 00000000..76bd771e --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerBuilderTest.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb; + +import org.junit.Test; +import org.springframework.util.Assert; + +/** + * @author agapple 2017年10月12日 上午10:50:00 + * @since 1.0.25 + */ +public class TableMetaManagerBuilderTest { + + @Test + public void testSimple() { + TableMetaTSDB tableMetaTSDB = TableMetaTSDBBuilder.build("test", "classpath:tsdb/mysql-tsdb.xml"); + Assert.notNull(tableMetaTSDB); + TableMetaTSDBBuilder.destory("test"); + } +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java index 13413d81..b17dd8ea 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java @@ -8,7 +8,6 @@ import java.net.URL; import javax.annotation.Resource; -import com.taobao.tddl.dbsync.binlog.BinlogPosition; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.junit.Test; @@ -16,12 +15,14 @@ import org.junit.runner.RunWith; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import com.alibaba.otter.canal.protocol.position.EntryPosition; + /** * @author wanshao 2017年8月2日 下午4:11:45 * @since 3.2.5 */ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = { "/dal-dao.xml" }) +@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" }) public class TableMetaManagerTest { @Resource @@ -29,17 +30,18 @@ public class TableMetaManagerTest { @Test public void testSimple() throws FileNotFoundException, IOException { + tableMetaManager.init("test"); URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); File dummyFile = new File(url.getFile()); File create = new File(dummyFile.getParent() + "/ddl", "create.sql"); - BinlogPosition position = BinlogPosition.parseFromString("001115:0139177334#3065927853.1501660815000"); + EntryPosition position = new EntryPosition("mysql-bin.001115", 139177334L, 3065927853L, 1501660815000L); String createSql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n"); - tableMetaManager.apply(position, "tddl5_00", createSql); + tableMetaManager.apply(position, "tddl5_00", createSql, null); String alterSql = "alter table `test` add column name varchar(32) after c_varchar"; - position = BinlogPosition.parseFromString("001115:0139177334#3065927853.1501660816000"); - tableMetaManager.apply(position, "tddl5_00", alterSql); + position = new EntryPosition("mysql-bin.001115", 139177334L, 3065927853L, 1501660815000L); + tableMetaManager.apply(position, "tddl5_00", alterSql, null); } } diff --git a/parse/src/test/resources/ddl/create.sql b/parse/src/test/resources/ddl/create.sql index 273be888..6f64a3a6 100644 --- a/parse/src/test/resources/ddl/create.sql +++ b/parse/src/test/resources/ddl/create.sql @@ -1,5 +1,5 @@ CREATE TABLE `test` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `id` bigint(20) zerofill unsigNed NOT NULL AUTO_INCREMENT COMMENT 'id', `c_tinyint` tinyint(4) DEFAULT '1' COMMENT 'tinyint', `c_smallint` smallint(6) DEFAULT 0 COMMENT 'smallint', `c_mediumint` mediumint(9) DEFAULT NULL COMMENT 'mediumint', diff --git a/parse/src/test/resources/tsdb/dal-dao.xml b/parse/src/test/resources/tsdb/dal-dao.xml deleted file mode 100644 index 1842641f..00000000 --- a/parse/src/test/resources/tsdb/dal-dao.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/parse/src/test/resources/tsdb/derby-tsdb.xml b/parse/src/test/resources/tsdb/derby-tsdb.xml new file mode 100644 index 00000000..a6c19486 --- /dev/null +++ b/parse/src/test/resources/tsdb/derby-tsdb.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/parse/src/test/resources/tsdb/mybatis-config.xml b/parse/src/test/resources/tsdb/mybatis-config.xml deleted file mode 100644 index 83ac0116..00000000 --- a/parse/src/test/resources/tsdb/mybatis-config.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/parse/src/test/resources/tsdb/mysql-tsdb.xml b/parse/src/test/resources/tsdb/mysql-tsdb.xml new file mode 100644 index 00000000..e3743109 --- /dev/null +++ b/parse/src/test/resources/tsdb/mysql-tsdb.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/parse/src/test/resources/tsdb/sqlmap-config.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap-config.xml similarity index 68% rename from parse/src/test/resources/tsdb/sqlmap-config.xml rename to parse/src/test/resources/tsdb/sql-map/sqlmap-config.xml index 983e9663..2485a35b 100644 --- a/parse/src/test/resources/tsdb/sqlmap-config.xml +++ b/parse/src/test/resources/tsdb/sql-map/sqlmap-config.xml @@ -3,7 +3,6 @@ "http://www.ibatis.com/dtd/sql-map-config-2.dtd"> - - - + + diff --git a/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml index 00197194..7445bf69 100644 --- a/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml +++ b/parse/src/test/resources/tsdb/sql-map/sqlmap_history.xml @@ -1,57 +1,45 @@ - - - - + + - insert into `canal_table_meta_history` () - values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#schema#,#table#,#sql#,#type#,#extra#); - - SELECT last_insert_id() - + insert into meta_history () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#useSchema#,#sqlSchema#,#sqlTable#,#sqlText#,#sqlType#,#extra#) + + + delete from meta_history + where destination=#destination# + - - - - \ No newline at end of file diff --git a/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml index 9065a7e0..7ba0a435 100644 --- a/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml +++ b/parse/src/test/resources/tsdb/sql-map/sqlmap_snapshot.xml @@ -1,66 +1,63 @@ - - - - - - + + + + - insert into `canal_table_meta_snapshot` () - values(now(),now(),#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#); - - SELECT last_insert_id() - + insert into meta_snapshot () + values(CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#destination#,#binlogFile#,#binlogOffest#,#binlogMasterId#,#binlogTimestamp#,#data#,#extra#) - update `canal_table_meta_snapshot` set gmt_modified=now(), + update meta_snapshot set gmt_modified=now(), binlog_file=#binlogFile#,binlog_offest=#binlogOffest#,binlog_master_id=#binlogMasterId#,binlog_timestamp=#binlogTimestamp#,data=#data#,extra=#extra# - where binlog_timestamp=0 + where destination=#destination# and binlog_timestamp=0 + + delete from meta_snapshot + where destination=#destination# + - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml index cb5f89cd..d1471058 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ 1.6 1.6 UTF-8 + 3.2.9.RELEASE @@ -131,14 +132,34 @@ - org.springframework - spring - 2.5.6 + org.springframework + spring-core + ${spring_version} + + + org.springframework + spring-aop + ${spring_version} + + + org.springframework + spring-context + ${spring_version} + + + org.springframework + spring-jdbc + ${spring_version} + + + org.springframework + spring-orm + ${spring_version} org.springframework spring-test - 2.5.6 + ${spring_version} test @@ -218,7 +239,7 @@ com.alibaba druid - 1.1.3 + 1.1.5-preview_14 @@ -251,9 +272,10 @@ mysql mysql-connector-java - 5.1.33 + 5.1.40 + diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java index 2890e9ad..1f8ec38d 100644 --- a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java +++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java @@ -1,5 +1,6 @@ package com.alibaba.otter.canal.protocol.position; + /** * 数据库对象的唯一标示 * @@ -121,4 +122,18 @@ public class EntryPosition extends TimePosition { return true; } + /** + * {@inheritDoc} + * + * @see java.lang.Comparable#compareTo(java.lang.Object) + */ + public int compareTo(EntryPosition o) { + final int val = journalName.compareTo(o.journalName); + + if (val == 0) { + return (int) (position - o.position); + } + return val; + } + } From e01e48ff658a566c8983a6487834f06c348cda14 Mon Sep 17 00:00:00 2001 From: agapple Date: Sat, 14 Oct 2017 01:14:13 +0800 Subject: [PATCH 04/34] fixed h2 support --- deployer/src/main/resources/canal.properties | 3 +- .../resources/example/instance.properties | 6 +- .../main/resources/spring/tsdb/h2-tsdb.xml | 60 +++++++++++++++++++ parse/pom.xml | 4 ++ .../inbound/mysql/tsdb/TableMetaManager.java | 5 +- .../inbound/mysql/tsdb/dao/MetaBaseDAO.java | 59 ++++++++++++++++++ .../mysql/tsdb/dao/MetaHistoryDAO.java | 34 +---------- .../mysql/tsdb/dao/MetaSnapshotDAO.java | 35 +---------- .../main/resources/ddl/h2/meta_history.sql | 21 +++++++ .../main/resources/ddl/h2/meta_snapshot.sql | 17 ++++++ .../parse/inbound/mysql/MysqlDumpTest.java | 2 +- .../mysql/tsdb/TableMetaManagerTest.java | 5 +- parse/src/test/resources/tsdb/h2-tsdb.xml | 44 ++++++++++++++ pom.xml | 5 ++ 14 files changed, 227 insertions(+), 73 deletions(-) create mode 100644 deployer/src/main/resources/spring/tsdb/h2-tsdb.xml create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaBaseDAO.java create mode 100644 parse/src/main/resources/ddl/h2/meta_history.sql create mode 100644 parse/src/main/resources/ddl/h2/meta_snapshot.sql create mode 100644 parse/src/test/resources/tsdb/h2-tsdb.xml diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 09616194..033a3f89 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -60,7 +60,8 @@ canal.conf.dir = ../conf canal.auto.scan = true canal.auto.scan.interval = 5 -canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml +canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml +#canal.instance.tsdb.spring.xml=classpath:spring/tsdb/mysql-tsdb.xml canal.instance.global.mode = spring canal.instance.global.lazy = false diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 50640d99..bef5cac3 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -4,13 +4,15 @@ canal.instance.mysql.slaveId=1234 # position info canal.instance.master.address=127.0.0.1:3306 canal.instance.master.journal.name=mysql-bin.000001 -canal.instance.master.position=68240 +canal.instance.master.position=104606 canal.instance.master.timestamp= # table meta tsdb info canal.instance.tsdb.enable=true -canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb +canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb canal.instance.tsdb.dbUsername=canal canal.instance.tsdb.dbPassword=canal diff --git a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml new file mode 100644 index 00000000..5c3b869d --- /dev/null +++ b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml @@ -0,0 +1,60 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/parse/pom.xml b/parse/pom.xml index 66e4bb60..0dd84d6f 100644 --- a/parse/pom.xml +++ b/parse/pom.xml @@ -58,6 +58,10 @@ org.apache.ibatis ibatis-sqlmap + + com.h2database + h2 + junit diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java index c8fe89e1..4e6a392d 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java @@ -46,10 +46,11 @@ public class TableMetaManager implements TableMetaTSDB { private static Logger logger = LoggerFactory.getLogger(TableMetaManager.class); private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); + private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); private static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L); private String destination; private MemoryTableMeta memoryTableMeta; - private MysqlConnection connection; // 查询meta信息的链接 + private MysqlConnection connection; // 查询meta信息的链接 private CanalEventFilter filter; private CanalEventFilter blackFilter; private EntryPosition lastPosition; @@ -466,7 +467,7 @@ public class TableMetaManager implements TableMetaTSDB { } public boolean isUkDuplicateException(Throwable t) { - if (pattern.matcher(t.getMessage()).find()) { + if (pattern.matcher(t.getMessage()).find() || h2Pattern.matcher(t.getMessage()).find()) { // 违反外键约束时也抛出这种异常，所以这里还要判断包含字符串Duplicate entry return true; } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaBaseDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaBaseDAO.java new file mode 100644 index 00000000..968e83b8 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaBaseDAO.java @@ -0,0 +1,59 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + +import javax.sql.DataSource; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; + +/** + * @author agapple 2017年10月14日 上午1:05:22 + * @since 1.0.25 + */ +@SuppressWarnings("deprecation") +public class MetaBaseDAO extends SqlMapClientDaoSupport { + + protected boolean isH2 = false; + + protected void initTable(String tableName) throws Exception { + Connection conn = null; + InputStream input = null; + try { + DataSource dataSource = getDataSource(); + conn = dataSource.getConnection(); + String name = "mysql"; + isH2 = isH2(conn); + if (isH2) { + name = "h2"; + } + input = Thread.currentThread() + .getContextClassLoader() + .getResourceAsStream("ddl/" + name + "/" + tableName + ".sql"); + if (input == null) { + return; + } + + String sql = StringUtils.join(IOUtils.readLines(input), "\n"); + Statement stmt = conn.createStatement(); + stmt.execute(sql); + stmt.close(); + } catch (Throwable e) { + logger.warn("init " + tableName + " failed", e); + } finally { + IOUtils.closeQuietly(input); + if (conn != null) { + conn.close(); + } + } + } + + private boolean isH2(Connection conn) throws SQLException { + String product = conn.getMetaData().getDatabaseProductName(); + return StringUtils.containsIgnoreCase(product, "H2"); + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java index e43c5e62..6ec51614 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaHistoryDAO.java @@ -1,19 +1,10 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; -import java.io.InputStream; -import java.sql.Connection; -import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.List; -import javax.sql.DataSource; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; - import com.google.common.collect.Maps; /** @@ -23,7 +14,7 @@ import com.google.common.collect.Maps; * @since 3.2.5 */ @SuppressWarnings("deprecation") -public class MetaHistoryDAO extends SqlMapClientDaoSupport { +public class MetaHistoryDAO extends MetaBaseDAO { public Long insert(MetaHistoryDO metaDO) { return (Long) getSqlMapClientTemplate().insert("meta_history.insert", metaDO); @@ -56,27 +47,6 @@ public class MetaHistoryDAO extends SqlMapClientDaoSupport { } protected void initDao() throws Exception { - Connection conn = null; - InputStream input = null; - try { - DataSource dataSource = getDataSource(); - conn = dataSource.getConnection(); - input = Thread.currentThread().getContextClassLoader().getResourceAsStream("ddl/mysql/meta_history.sql"); - if (input == null) { - return; - } - - String sql = StringUtils.join(IOUtils.readLines(input), "\n"); - Statement stmt = conn.createStatement(); - stmt.execute(sql); - stmt.close(); - } catch (Throwable e) { - logger.warn("init meta_history failed", e); - } finally { - IOUtils.closeQuietly(input); - if (conn != null) { - conn.close(); - } - } + initTable("meta_history"); } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java index 1a3f3053..f525031a 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/dao/MetaSnapshotDAO.java @@ -1,18 +1,9 @@ package com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao; -import java.io.InputStream; -import java.sql.Connection; -import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; -import javax.sql.DataSource; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.springframework.orm.ibatis.support.SqlMapClientDaoSupport; - import com.google.common.collect.Maps; /** @@ -22,7 +13,7 @@ import com.google.common.collect.Maps; * @since 3.2.5 */ @SuppressWarnings("deprecation") -public class MetaSnapshotDAO extends SqlMapClientDaoSupport { +public class MetaSnapshotDAO extends MetaBaseDAO { public Long insert(MetaSnapshotDO snapshotDO) { return (Long) getSqlMapClientTemplate().insert("meta_snapshot.insert", snapshotDO); @@ -59,27 +50,7 @@ public class MetaSnapshotDAO extends SqlMapClientDaoSupport { } protected void initDao() throws Exception { - Connection conn = null; - InputStream input = null; - try { - DataSource dataSource = getDataSource(); - conn = dataSource.getConnection(); - input = Thread.currentThread().getContextClassLoader().getResourceAsStream("ddl/mysql/meta_snapshot.sql"); - if (input == null) { - return; - } - - String sql = StringUtils.join(IOUtils.readLines(input), "\n"); - Statement stmt = conn.createStatement(); - stmt.execute(sql); - stmt.close(); - } catch (Throwable e) { - logger.warn("init meta_history failed", e); - } finally { - IOUtils.closeQuietly(input); - if (conn != null) { - conn.close(); - } - } + initTable("meta_snapshot"); } + } diff --git a/parse/src/main/resources/ddl/h2/meta_history.sql b/parse/src/main/resources/ddl/h2/meta_history.sql new file mode 100644 index 00000000..4c7bbff4 --- /dev/null +++ b/parse/src/main/resources/ddl/h2/meta_history.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS `meta_history` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `use_schema` varchar(1024) DEFAULT NULL COMMENT '执行sql时对应的schema', + `sql_schema` varchar(1024) DEFAULT NULL COMMENT '对应的schema', + `sql_table` varchar(1024) DEFAULT NULL COMMENT '对应的table', + `sql_text` longtext DEFAULT NULL COMMENT '执行的sql', + `sql_type` varchar(256) DEFAULT NULL COMMENT 'sql类型', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY meta_history_binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `meta_history_destination` (`destination`), + KEY `meta_history_destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `meta_history_gmt_modified` (`gmt_modified`) +); \ No newline at end of file diff --git a/parse/src/main/resources/ddl/h2/meta_snapshot.sql b/parse/src/main/resources/ddl/h2/meta_snapshot.sql new file mode 100644 index 00000000..c14d7358 --- /dev/null +++ b/parse/src/main/resources/ddl/h2/meta_snapshot.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS `meta_snapshot` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键', + `gmt_create` datetime NOT NULL COMMENT '创建时间', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `destination` varchar(128) DEFAULT NULL COMMENT '通道名称', + `binlog_file` varchar(64) DEFAULT NULL COMMENT 'binlog文件名', + `binlog_offest` bigint(20) DEFAULT NULL COMMENT 'binlog偏移量', + `binlog_master_id` varchar(64) DEFAULT NULL COMMENT 'binlog节点id', + `binlog_timestamp` bigint(20) DEFAULT NULL COMMENT 'binlog应用的时间戳', + `data` longtext DEFAULT NULL COMMENT '表结构数据', + `extra` text DEFAULT NULL COMMENT '额外的扩展信息', + PRIMARY KEY (`id`), + UNIQUE KEY meta_snapshot_binlog_file_offest(`destination`,`binlog_master_id`,`binlog_file`,`binlog_offest`), + KEY `meta_snapshot_destination` (`destination`), + KEY `meta_snapshot_destination_timestamp` (`destination`,`binlog_timestamp`), + KEY `meta_snapshot_gmt_modified` (`gmt_modified`) +); \ No newline at end of file diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java index aef55822..a0754635 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java @@ -36,7 +36,7 @@ public class MysqlDumpTest { controller.setMasterPosition(startPosition); controller.setEnableTsdb(true); controller.setDestination("example"); - controller.setTsdbSpringXml("classpath:tsdb/mysql-tsdb.xml"); + controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml"); controller.setEventFilter(new AviaterRegexFilter("test\\..*")); controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*")); controller.setEventSink(new AbstractCanalEventSinkTest>() { diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java index b17dd8ea..af8c9dc5 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java @@ -22,7 +22,7 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition; * @since 3.2.5 */ @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(locations = { "/tsdb/mysql-tsdb.xml" }) +@ContextConfiguration(locations = { "/tsdb/h2-tsdb.xml" }) public class TableMetaManagerTest { @Resource @@ -40,8 +40,7 @@ public class TableMetaManagerTest { tableMetaManager.apply(position, "tddl5_00", createSql, null); String alterSql = "alter table `test` add column name varchar(32) after c_varchar"; - position = new EntryPosition("mysql-bin.001115", 139177334L, 3065927853L, 1501660815000L); + position = new EntryPosition("mysql-bin.001115", 139177334L, 3065927854L, 1501660815000L); tableMetaManager.apply(position, "tddl5_00", alterSql, null); - } } diff --git a/parse/src/test/resources/tsdb/h2-tsdb.xml b/parse/src/test/resources/tsdb/h2-tsdb.xml new file mode 100644 index 00000000..452e7ee5 --- /dev/null +++ b/parse/src/test/resources/tsdb/h2-tsdb.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index d1471058..649abb36 100644 --- a/pom.xml +++ b/pom.xml @@ -262,6 +262,11 @@ slf4j-api 1.7.12 + + com.h2database + h2 + 1.4.196 + junit From 77342663e3fc8ec4b4dd465aa95d998d411c2d90 Mon Sep 17 00:00:00 2001 From: agapple Date: Sun, 15 Oct 2017 15:55:30 +0800 Subject: [PATCH 05/34] fixed rds binlog local --- common/pom.xml | 4 + .../monitor/SpringInstanceConfigMonitor.java | 4 +- deployer/src/main/resources/canal.properties | 1 + .../resources/example/instance.properties | 10 +- .../resources/example/rds_instance.properties | 31 ++ .../resources/spring/default-instance.xml | 4 +- .../main/resources/spring/file-instance.xml | 4 +- .../main/resources/spring/group-instance.xml | 4 +- .../main/resources/spring/local-instance.xml | 63 +-- .../main/resources/spring/memory-instance.xml | 2 +- .../main/resources/spring/tsdb/h2-tsdb.xml | 2 +- .../main/resources/spring/tsdb/mysql-tsdb.xml | 2 +- parse/pom.xml | 8 + .../inbound/mysql/LocalBinLogConnection.java | 68 ++-- .../inbound/mysql/LocalBinlogEventParser.java | 21 +- .../parse/inbound/mysql/MysqlConnection.java | 1 + .../parse/inbound/mysql/MysqlEventParser.java | 37 +- .../inbound/mysql/dbsync/LogEventConvert.java | 44 ++- .../inbound/mysql/ddl/DruidDdlParser.java | 6 +- .../parse/inbound/mysql/rds/HttpHelper.java | 368 ++++++++++++++++++ .../inbound/mysql/rds/RdsBinlogOpenApi.java | 334 ++++++++++++++++ .../mysql/rds/RdsLocalBinlogEventParser.java | 111 ++++++ ...etaManager.java => DatabaseTableMeta.java} | 8 +- .../inbound/mysql/RdsBinlogOpenApiTest.java | 27 ++ .../inbound/mysql/RdsLocalBinlogDumpTest.java | 117 ++++++ .../mysql/tsdb/TableMetaManagerTest.java | 2 +- parse/src/test/resources/tsdb/derby-tsdb.xml | 2 +- parse/src/test/resources/tsdb/h2-tsdb.xml | 2 +- parse/src/test/resources/tsdb/mysql-tsdb.xml | 2 +- pom.xml | 15 + 30 files changed, 1174 insertions(+), 130 deletions(-) create mode 100644 deployer/src/main/resources/example/rds_instance.properties create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java rename parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/{TableMetaManager.java => DatabaseTableMeta.java} (98%) create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsLocalBinlogDumpTest.java diff --git a/common/pom.xml b/common/pom.xml index 4789ac9e..eb702d71 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -35,6 +35,10 @@ commons-lang commons-lang + + commons-codec + commons-codec + com.alibaba fastjson diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java index 38d6d24c..3483324d 100644 --- a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java @@ -208,8 +208,7 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen action.reload(destination); logger.info("auto notify reload {} successful.", destination); } catch (Throwable e) { - logger.error(String.format("scan reload found[%s] but reload failed", - destination), e); + logger.error(String.format("scan reload found[%s] but reload failed", destination), e); } } } @@ -312,5 +311,4 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen } } - } diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties index 033a3f89..f765e71b 100644 --- a/deployer/src/main/resources/canal.properties +++ b/deployer/src/main/resources/canal.properties @@ -66,6 +66,7 @@ canal.instance.tsdb.spring.xml=classpath:spring/tsdb/h2-tsdb.xml canal.instance.global.mode = spring canal.instance.global.lazy = false #canal.instance.global.manager.address = 127.0.0.1:1099 +#canal.instance.global.spring.xml = classpath:spring/local-instance.xml #canal.instance.global.spring.xml = classpath:spring/memory-instance.xml canal.instance.global.spring.xml = classpath:spring/file-instance.xml #canal.instance.global.spring.xml = classpath:spring/default-instance.xml diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index bef5cac3..31b625de 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -1,10 +1,10 @@ ################################################# ## mysql serverId -canal.instance.mysql.slaveId=1234 +canal.instance.mysql.slaveId=0 # position info canal.instance.master.address=127.0.0.1:3306 -canal.instance.master.journal.name=mysql-bin.000001 -canal.instance.master.position=104606 +canal.instance.master.journal.name= +canal.instance.master.position= canal.instance.master.timestamp= @@ -27,7 +27,7 @@ canal.instance.dbPassword=canal canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 # table regex -canal.instance.filter.regex=test\\..* +canal.instance.filter.regex=*\\..* # table black regex -canal.instance.filter.black.regex=canal_tsdb\\..* +canal.instance.filter.black.regex= ################################################# \ No newline at end of file diff --git a/deployer/src/main/resources/example/rds_instance.properties b/deployer/src/main/resources/example/rds_instance.properties new file mode 100644 index 00000000..9712cf93 --- /dev/null +++ b/deployer/src/main/resources/example/rds_instance.properties @@ -0,0 +1,31 @@ +################################################# +canal.instance.rds.open.url=https://rds.aliyuncs.com/ +canal.instance.rds.open.accesskey= +canal.instance.rds.open.secretkey= +canal.instance.rds.instanceId= +canal.instance.rds.startTime= +canal.instance.rds.endTime= + +# local binlog dir +canal.instance.parser.directory=${canal.file.data.dir:../conf}/${canal.instance.destination:}/binlog +# position info +canal.instance.master.address=127.0.0.1:3306 + +# table meta tsdb info +canal.instance.tsdb.enable=true +canal.instance.tsdb.dir=${canal.file.data.dir:../conf}/${canal.instance.destination:} +canal.instance.tsdb.url=jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL; +#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb +#canal.instance.tsdb.dbUsername=canal +#canal.instance.tsdb.dbPassword=canal + +# username/password +canal.instance.dbUsername=canal +canal.instance.dbPassword=canal +canal.instance.defaultDatabaseName=test +canal.instance.connectionCharset=UTF-8 +# table regex +canal.instance.filter.regex=test\\..* +# table black regex +canal.instance.filter.black.regex= +################################################# \ No newline at end of file diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml index 9c7ef5aa..8a26c34a 100644 --- a/deployer/src/main/resources/spring/default-instance.xml +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -85,7 +85,7 @@ - + @@ -185,7 +185,7 @@ - + diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml index 4c6f750d..3ee38c98 100644 --- a/deployer/src/main/resources/spring/file-instance.xml +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -71,7 +71,7 @@ - + @@ -170,7 +170,7 @@ - + diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml index f5e872f3..5ac91794 100644 --- a/deployer/src/main/resources/spring/group-instance.xml +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -77,7 +77,7 @@ - + @@ -168,7 +168,7 @@ - + diff --git a/deployer/src/main/resources/spring/local-instance.xml b/deployer/src/main/resources/spring/local-instance.xml index e2fb44b3..5ac62df1 100644 --- a/deployer/src/main/resources/spring/local-instance.xml +++ b/deployer/src/main/resources/spring/local-instance.xml @@ -48,41 +48,14 @@ - - - - - - - - - - - - - - + - - - - - - ${canal.zkServers:127.0.0.1:2181} - - - - - - - - - - - + + + @@ -96,10 +69,10 @@ - + - + @@ -144,25 +117,25 @@ - - - - - - - - - - + + + + + + + + + + - + - + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml index 129ce2b0..7f1eff3f 100644 --- a/deployer/src/main/resources/spring/memory-instance.xml +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -68,7 +68,7 @@ - + diff --git a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml index 5c3b869d..6f8ba05d 100644 --- a/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml +++ b/deployer/src/main/resources/spring/tsdb/h2-tsdb.xml @@ -23,7 +23,7 @@ - + diff --git a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml index 5f02431d..555eec44 100644 --- a/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml +++ b/deployer/src/main/resources/spring/tsdb/mysql-tsdb.xml @@ -23,7 +23,7 @@ - + diff --git a/parse/pom.xml b/parse/pom.xml index 0dd84d6f..f495b52c 100644 --- a/parse/pom.xml +++ b/parse/pom.xml @@ -62,6 +62,14 @@ com.h2database h2 + + org.apache.httpcomponents + httpclient + + + org.apache.commons + commons-compress + junit diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java index 3c91350a..73b4f3d1 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java @@ -8,6 +8,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.ErosaConnection; import com.alibaba.otter.canal.parse.inbound.SinkFunction; import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue; @@ -84,26 +85,19 @@ public class LocalBinLogConnection implements ErosaConnection { while (running) { boolean needContinue = true; LogEvent event = null; - L: while (fetcher.fetch()) { - /* - * event = decoder.decode(fetcher, context); if (event == - * null) { throw new CanalParseException("parse failed"); } - * if (!func.sink(event)) { needContinue = false; break; } - */ + // 处理一下binlog文件名 + event = new RotateLogEvent(context.getLogPosition().getFileName(), 4); + func.sink(event); + while (fetcher.fetch()) { + event = decoder.decode(fetcher, context); + if (event == null) { + throw new CanalParseException("parse failed"); + } - do { - if (event == null) { - event = new RotateLogEvent(context.getLogPosition().getFileName(), context.getLogPosition() - .getPosition()); - } else { - event = decoder.decode(fetcher, context); - } - - if (event != null && !func.sink(event)) { - needContinue = false; - break L; - } - } while (event != null); + if (!func.sink(event)) { + needContinue = false; + break; + } } if (needContinue) {// 读取下一个 @@ -121,7 +115,6 @@ public class LocalBinLogConnection implements ErosaConnection { } current = nextFile; - fetcher.open(current); context.setLogPosition(new LogPosition(nextFile.getName())); } else { @@ -161,30 +154,27 @@ public class LocalBinLogConnection implements ErosaConnection { binlogFilename = lastXidLogFilename; binlogFileOffset = lastXidLogFileOffset; - L: while (fetcher.fetch()) { - LogEvent event; - do { - event = decoder.decode(fetcher, context); - if (event != null) { - if (event.getWhen() > timestampSeconds) { - break L; - } + while (fetcher.fetch()) { + LogEvent event = decoder.decode(fetcher, context); + if (event != null) { + if (event.getWhen() > timestampSeconds) { + break; + } - needContinue = false; - if (LogEvent.QUERY_EVENT == event.getHeader().getType()) { - if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) { - binlogFilename = lastXidLogFilename; - binlogFileOffset = lastXidLogFileOffset; - } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) { - lastXidLogFilename = current.getName(); - lastXidLogFileOffset = event.getLogPos(); - } - } else if (LogEvent.XID_EVENT == event.getHeader().getType()) { + needContinue = false; + if (LogEvent.QUERY_EVENT == event.getHeader().getType()) { + if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) { + binlogFilename = lastXidLogFilename; + binlogFileOffset = lastXidLogFileOffset; + } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) { lastXidLogFilename = current.getName(); lastXidLogFileOffset = event.getLogPos(); } + } else if (LogEvent.XID_EVENT == event.getHeader().getType()) { + lastXidLogFilename = current.getName(); + lastXidLogFileOffset = event.getLogPos(); } - } while (event != null); + } } if (needContinue) {// 读取下一个 diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java index d2c2fd2e..ea6b0365 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java @@ -9,6 +9,7 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.ErosaConnection; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta; import com.alibaba.otter.canal.parse.index.CanalLogPositionManager; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.position.EntryPosition; @@ -23,14 +24,14 @@ import com.alibaba.otter.canal.protocol.position.LogPosition; public class LocalBinlogEventParser extends AbstractMysqlEventParser implements CanalEventParser { // 数据库信息 - private AuthenticationInfo masterInfo; - private EntryPosition masterPosition; // binlog信息 - private MysqlConnection metaConnection; // 查询meta信息的链接 - private TableMetaCache tableMetaCache; // 对应meta + protected AuthenticationInfo masterInfo; + protected EntryPosition masterPosition; // binlog信息 + protected MysqlConnection metaConnection; // 查询meta信息的链接 + protected TableMetaCache tableMetaCache; // 对应meta - private String directory; - private boolean needWait = false; - private int bufferSize = 16 * 1024; + protected String directory; + protected boolean needWait = false; + protected int bufferSize = 16 * 1024; public LocalBinlogEventParser(){ // this.runningInfo = new AuthenticationInfo(); @@ -50,6 +51,12 @@ public class LocalBinlogEventParser extends AbstractMysqlEventParser implements throw new CanalParseException(e); } + if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) { + ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection); + ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter); + ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); + } + tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 94625b10..7b141010 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -156,6 +156,7 @@ public class MysqlConnection implements ErosaConnection { BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket(); binlogDumpCmd.binlogFileName = binlogfilename; binlogDumpCmd.binlogPosition = binlogPosition; + // binlogDumpCmd.slaveServerId = this.slaveId; binlogDumpCmd.slaveServerId = this.slaveId; byte[] cmdBody = binlogDumpCmd.toBytes(); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 912787b4..efee2b71 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -1,7 +1,9 @@ package com.alibaba.otter.canal.parse.inbound.mysql; import java.io.IOException; +import java.net.InetAddress; import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,7 +28,7 @@ import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogFormat; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection.BinlogImage; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache; -import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaManager; +import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.DatabaseTableMeta; import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.position.EntryPosition; @@ -118,10 +120,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } } - if (tableMetaTSDB != null && tableMetaTSDB instanceof TableMetaManager) { - ((TableMetaManager) tableMetaTSDB).setConnection(metaConnection); - ((TableMetaManager) tableMetaTSDB).setFilter(eventFilter); - ((TableMetaManager) tableMetaTSDB).setBlackFilter(eventBlackFilter); + if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) { + ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection); + ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter); + ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter); } tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB); @@ -313,10 +315,35 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE connection.getConnector().setSendBufferSize(sendBufferSize); connection.getConnector().setSoTimeout(defaultConnectionTimeoutInSeconds * 1000); connection.setCharset(connectionCharset); + // 随机生成slaveId + if (this.slaveId <= 0) { + this.slaveId = generateUniqueServerId(); + } connection.setSlaveId(this.slaveId); return connection; } + private final long generateUniqueServerId() { + try { + // a=`echo $masterip|cut -d\. -f1` + // b=`echo $masterip|cut -d\. -f2` + // c=`echo $masterip|cut -d\. -f3` + // d=`echo $masterip|cut -d\. -f4` + // #server_id=`expr $a \* 256 \* 256 \* 256 + $b \* 256 \* 256 + $c + // \* 256 + $d ` + // #server_id=$b$c$d + // server_id=`expr $b \* 256 \* 256 + $c \* 256 + $d ` + InetAddress localHost = InetAddress.getLocalHost(); + byte[] addr = localHost.getAddress(); + int salt = (destination != null) ? destination.hashCode() : 0; + return ((0x7f & salt) << 24) + ((0xff & (int) addr[1]) << 16) // NL + + ((0xff & (int) addr[2]) << 8) // NL + + (0xff & (int) addr[3]); + } catch (UnknownHostException e) { + throw new CanalParseException("Unknown host", e); + } + } + protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException { EntryPosition startPosition = findStartPositionInternal(connection); if (needTransactionPosition.get()) { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 538f6b95..34474efd 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -6,6 +6,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.Charset; import java.sql.Types; +import java.util.Arrays; import java.util.BitSet; import java.util.List; @@ -294,7 +295,15 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar String queryString = null; try { queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name()); - return buildQueryEntry(queryString, event.getHeader()); + String tableName = null; + if (useDruidDdlFilter) { + List results = DruidDdlParser.parse(queryString, null); + if (results.size() > 0) { + tableName = results.get(0).getTableName(); + } + } + + return buildQueryEntry(queryString, event.getHeader(), tableName); } catch (UnsupportedEncodingException e) { throw new CanalParseException(e); } @@ -355,6 +364,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar throw new TableIdNotFoundException("not found tableId:" + event.getTableId()); } + boolean isRDSHeartBeat = tableMetaCache.isOnRDS() + && isRDSHeartBeat(table.getDbName(), table.getTableName()); + String fullname = table.getDbName() + "." + table.getTableName(); // check name filter if (nameFilter != null && !nameFilter.filter(fullname)) { @@ -364,9 +376,17 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return null; } - if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) { - // 忽略rds模式的mysql.ha_health_check心跳数据 - return null; + // if (isHeartBeat || isRDSHeartBeat) { + // // 忽略rds模式的mysql.ha_health_check心跳数据 + // return null; + // } + TableMeta tableMeta = null; + if (isRDSHeartBeat) { + // 处理rds模式的mysql.ha_health_check心跳数据 + // 主要RDS的心跳表基本无权限,需要mock一个tableMeta + FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0"); + FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0"); + tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta)); } EventType eventType = null; @@ -396,8 +416,8 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar BitSet columns = event.getColumns(); BitSet changeColumns = event.getChangeColumns(); boolean tableError = false; - TableMeta tableMeta = null; - if (tableMetaCache != null) {// 入错存在table meta cache + if (tableMetaCache != null && tableMeta == null) {// 入错存在table meta + // cache tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position); if (tableMeta == null) { tableError = true; @@ -661,6 +681,14 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar } + private Entry buildQueryEntry(String queryString, LogHeader logHeader, String tableName) { + Header header = createHeader(binlogFileName, logHeader, "", tableName, EventType.QUERY); + RowChange.Builder rowChangeBuider = RowChange.newBuilder(); + rowChangeBuider.setSql(queryString); + rowChangeBuider.setEventType(EventType.QUERY); + return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString()); + } + private Entry buildQueryEntry(String queryString, LogHeader logHeader) { Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY); RowChange.Builder rowChangeBuider = RowChange.newBuilder(); @@ -739,6 +767,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType); } + private boolean isRDSHeartBeat(String schema, String table) { + return "mysql".equalsIgnoreCase(schema) && "ha_health_check".equalsIgnoreCase(table); + } + public static TransactionBegin createTransactionBegin(long threadId) { TransactionBegin.Builder beginBuilder = TransactionBegin.newBuilder(); beginBuilder.setThreadId(threadId); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java index 15577298..b2f4fa01 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java @@ -140,21 +140,21 @@ public class DruidDdlParser { } else if (statement instanceof SQLInsertStatement) { DdlResult ddlResult = new DdlResult(); SQLInsertStatement insert = (SQLInsertStatement) statement; - processName(ddlResult, schmeaName, insert.getTableName(), true); + processName(ddlResult, schmeaName, insert.getTableName(), false); ddlResult.setType(EventType.INSERT); ddlResults.add(ddlResult); } else if (statement instanceof SQLUpdateStatement) { DdlResult ddlResult = new DdlResult(); SQLUpdateStatement update = (SQLUpdateStatement) statement; // 拿到的表名可能为null,比如update a,b set a.id=x - processName(ddlResult, schmeaName, update.getTableName(), true); + processName(ddlResult, schmeaName, update.getTableName(), false); ddlResult.setType(EventType.UPDATE); ddlResults.add(ddlResult); } else if (statement instanceof SQLDeleteStatement) { DdlResult ddlResult = new DdlResult(); SQLDeleteStatement delete = (SQLDeleteStatement) statement; // 拿到的表名可能为null,比如delete a,b from a where a.id = b.id - processName(ddlResult, schmeaName, delete.getTableName(), true); + processName(ddlResult, schmeaName, delete.getTableName(), false); ddlResult.setType(EventType.DELETE); ddlResults.add(ddlResult); } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java new file mode 100644 index 00000000..7b641ade --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/HttpHelper.java @@ -0,0 +1,368 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.rds; + +import static org.apache.http.client.config.RequestConfig.custom; + +import java.io.IOException; +import java.net.URI; +import java.nio.charset.Charset; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.net.ssl.SSLContext; + +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.CookieStore; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.cookie.Cookie; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.TrustStrategy; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +public class HttpHelper { + + protected static final Logger logger = LoggerFactory.getLogger(HttpHelper.class); + + public static byte[] getBytes(String url, int timeout) throws Exception { + long start = System.currentTimeMillis(); + HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setMaxConnPerRoute(50); + builder.setMaxConnTotal(100); + CloseableHttpClient httpclient = builder.build(); + URI uri = new URIBuilder(url).build(); + RequestConfig config = custom().setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout) + .build(); + HttpGet httpGet = new HttpGet(uri); + HttpClientContext context = HttpClientContext.create(); + context.setRequestConfig(config); + CloseableHttpResponse response = httpclient.execute(httpGet, context); + try { + int statusCode = response.getStatusLine().getStatusCode(); + long end = System.currentTimeMillis(); + long cost = end - start; + if (logger.isWarnEnabled()) { + logger.warn("post " + url + ", cost : " + cost); + } + if (statusCode == HttpStatus.SC_OK) { + return EntityUtils.toByteArray(response.getEntity()); + } else { + String errorMsg = EntityUtils.toString(response.getEntity()); + throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode + + ", error msg=" + errorMsg); + } + } finally { + response.close(); + httpGet.releaseConnection(); + } + } + + public static String get(String url, int timeout) { + // logger.info("get url is :" + url); + // 支持采用https协议，忽略证书 + url = url.trim(); + if (url.startsWith("https")) { + return getIgnoreCerf(url, null, null, timeout); + } + long start = System.currentTimeMillis(); + HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setMaxConnPerRoute(50); + builder.setMaxConnTotal(100); + CloseableHttpClient httpclient = builder.build(); + CloseableHttpResponse response = null; + HttpGet httpGet = null; + try { + URI uri = new URIBuilder(url).build(); + RequestConfig config = custom().setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout) + .build(); + httpGet = new HttpGet(uri); + HttpClientContext context = HttpClientContext.create(); + context.setRequestConfig(config); + response = httpclient.execute(httpGet, context); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_OK) { + return EntityUtils.toString(response.getEntity()); + } else { + String errorMsg = EntityUtils.toString(response.getEntity()); + throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode + + ", error msg=" + errorMsg); + } + } catch (Throwable t) { + long end = System.currentTimeMillis(); + long cost = end - start; + String curlRequest = getCurlRequest(url, null, null, cost); + throw new RuntimeException("requestGet remote error, request : " + curlRequest, t); + } finally { + long end = System.currentTimeMillis(); + long cost = end - start; + printCurlRequest(url, null, null, cost); + if (response != null) { + try { + response.close(); + } catch (IOException e) { + } + } + httpGet.releaseConnection(); + } + } + + private static String getIgnoreCerf(String url, CookieStore cookieStore, Map params, int timeout) { + long start = System.currentTimeMillis(); + HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setMaxConnPerRoute(50); + builder.setMaxConnTotal(100); + HttpGet httpGet = null; + CloseableHttpResponse response = null; + try { + // 创建支持忽略证书的https + final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { + + @Override + public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + return true; + } + }).build(); + + CloseableHttpClient httpClient = HttpClientBuilder.create() + .setSSLContext(sslContext) + .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder. create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)) + .build())) + .build(); + // ---------------- 创建支持https 的client成功--------- + + URI uri = new URIBuilder(url).build(); + RequestConfig config = custom().setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout) + .build(); + httpGet = new HttpGet(uri); + HttpClientContext context = HttpClientContext.create(); + context.setRequestConfig(config); + response = httpClient.execute(httpGet, context); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_OK) { + return EntityUtils.toString(response.getEntity()); + } else { + String errorMsg = EntityUtils.toString(response.getEntity()); + throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode + + ", error msg=" + errorMsg); + } + } catch (Throwable t) { + long end = System.currentTimeMillis(); + long cost = end - start; + String curlRequest = getCurlRequest(url, cookieStore, params, cost); + throw new RuntimeException("requestPost(Https) remote error, request : " + curlRequest, t); + } finally { + long end = System.currentTimeMillis(); + long cost = end - start; + printCurlRequest(url, null, null, cost); + if (response != null) { + try { + response.close(); + } catch (IOException e) { + } + } + if (httpGet != null) { + httpGet.releaseConnection(); + } + } + } + + private static String postIgnoreCerf(String url, CookieStore cookieStore, Map params, int timeout) { + long start = System.currentTimeMillis(); + HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setMaxConnPerRoute(50); + builder.setMaxConnTotal(100); + HttpPost httpPost = null; + CloseableHttpResponse response = null; + try { + // 创建支持忽略证书的https + final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() { + + @Override + public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { + return true; + } + }).build(); + + CloseableHttpClient httpClient = HttpClientBuilder.create() + .setSSLContext(sslContext) + .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder. create() + .register("http", PlainConnectionSocketFactory.INSTANCE) + .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE)) + .build())) + .build(); + // ---------------- 创建支持https 的client成功--------- + + URI uri = new URIBuilder(url).build(); + RequestConfig config = custom().setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout) + .build(); + httpPost = new HttpPost(uri); + List parameters = Lists.newArrayList(); + for (String key : params.keySet()) { + NameValuePair nameValuePair = new BasicNameValuePair(key, params.get(key)); + parameters.add(nameValuePair); + } + httpPost.setEntity(new UrlEncodedFormEntity(parameters, Charset.forName("UTF-8"))); + HttpClientContext context = HttpClientContext.create(); + context.setRequestConfig(config); + context.setCookieStore(cookieStore); + + response = httpClient.execute(httpPost, context); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_OK) { + long end = System.currentTimeMillis(); + long cost = end - start; + printCurlRequest(url, cookieStore, params, cost); + return EntityUtils.toString(response.getEntity()); + } else { + long end = System.currentTimeMillis(); + long cost = end - start; + String curlRequest = getCurlRequest(url, cookieStore, params, cost); + throw new RuntimeException("requestPost(Https) remote error, request : " + curlRequest + + ", statusCode=" + statusCode + ""); + } + } catch (Throwable t) { + long end = System.currentTimeMillis(); + long cost = end - start; + String curlRequest = getCurlRequest(url, cookieStore, params, cost); + throw new RuntimeException("requestPost(Https) remote error, request : " + curlRequest, t); + } finally { + if (response != null) { + try { + response.close(); + } catch (IOException e) { + } + } + if (httpPost != null) { + httpPost.releaseConnection(); + } + } + } + + public static String post(String url, CookieStore cookieStore, Map params, int timeout) { + url = url.trim(); + // 支持采用https协议，忽略证书 + if (url.startsWith("https")) { + return postIgnoreCerf(url, cookieStore, params, timeout); + } + long start = System.currentTimeMillis(); + HttpClientBuilder builder = HttpClientBuilder.create(); + builder.setMaxConnPerRoute(50); + builder.setMaxConnTotal(100); + HttpPost httpPost = null; + CloseableHttpResponse response = null; + try { + CloseableHttpClient httpclient = builder.build(); + URI uri = new URIBuilder(url).build(); + RequestConfig config = custom().setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setSocketTimeout(timeout) + .build(); + httpPost = new HttpPost(uri); + List parameters = Lists.newArrayList(); + for (String key : params.keySet()) { + NameValuePair nameValuePair = new BasicNameValuePair(key, params.get(key)); + parameters.add(nameValuePair); + } + httpPost.setEntity(new UrlEncodedFormEntity(parameters, Charset.forName("UTF-8"))); + HttpClientContext context = HttpClientContext.create(); + context.setRequestConfig(config); + context.setCookieStore(cookieStore); + + response = httpclient.execute(httpPost, context); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == HttpStatus.SC_OK) { + long end = System.currentTimeMillis(); + long cost = end - start; + printCurlRequest(url, cookieStore, params, cost); + return EntityUtils.toString(response.getEntity()); + } else { + long end = System.currentTimeMillis(); + long cost = end - start; + String curlRequest = getCurlRequest(url, cookieStore, params, cost); + throw new RuntimeException("requestPost remote error, request : " + curlRequest + ", statusCode=" + + statusCode + ";" + EntityUtils.toString(response.getEntity())); + } + } catch (Throwable t) { + long end = System.currentTimeMillis(); + long cost = end - start; + String curlRequest = getCurlRequest(url, cookieStore, params, cost); + throw new RuntimeException("requestPost remote error, request : " + curlRequest, t); + } finally { + if (response != null) { + try { + response.close(); + } catch (IOException e) { + } + } + if (httpPost != null) { + httpPost.releaseConnection(); + } + } + } + + public static void printCurlRequest(String url, CookieStore cookieStore, Map params, long cost) { + logger.warn(getCurlRequest(url, cookieStore, params, cost)); + } + + private static String getCurlRequest(String url, CookieStore cookieStore, Map params, long cost) { + if (params == null) { + return "curl '" + url + "'\ncost : " + cost; + } else { + StringBuilder paramsStr = new StringBuilder(); + Iterator> iterator = params.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + paramsStr.append(entry.getKey() + "=" + entry.getValue()); + if (iterator.hasNext()) { + paramsStr.append("&"); + } + } + if (cookieStore == null) { + return "curl '" + url + "' -d '" + paramsStr.toString() + "'\ncost : " + cost; + } else { + StringBuilder cookieStr = new StringBuilder(); + List cookies = cookieStore.getCookies(); + Iterator iteratorCookie = cookies.iterator(); + while (iteratorCookie.hasNext()) { + Cookie cookie = iteratorCookie.next(); + cookieStr.append(cookie.getName() + "=" + cookie.getValue()); + if (iteratorCookie.hasNext()) { + cookieStr.append(";"); + } + } + return "curl '" + url + "' -b '" + cookieStr + "' -d '" + paramsStr.toString() + "'\ncost : " + cost; + } + } + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java new file mode 100644 index 00000000..eb638739 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsBinlogOpenApi.java @@ -0,0 +1,334 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.rds; + +import io.netty.handler.codec.http.HttpResponseStatus; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.UUID; + +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * @author agapple 2017年10月14日 下午1:53:52 + * @since 1.0.25 + */ +public class RdsBinlogOpenApi { + + protected static final Logger logger = LoggerFactory.getLogger(RdsBinlogOpenApi.class); + private static final String ISO8601_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + private static final int TIMEOUT = 10000; + private static final String ENCODING = "UTF-8"; + private static final String MAC_NAME = "HmacSHA1"; + private static final String API_VERSION = "2014-08-15"; + private static final String SIGNATURE_VERSION = "1.0"; + + public static void downloadBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime, + Date endTime, File destDir) throws Throwable { + int pageSize = 100; + int pageNumber = 0; + int pageRecordCount = 1; + String hostInstanceID = null; + while (pageRecordCount > 0 && pageRecordCount <= pageSize) { + pageNumber += 1; + String result = describeBinlogFiles(url, ak, sk, dbInstanceId, startTime, endTime, pageSize, pageNumber); + JSONObject jsobObj = JSON.parseObject(result); + pageRecordCount = jsobObj.getInteger("PageRecordCount"); + + if (pageRecordCount > 0) { + FileUtils.forceMkdir(destDir); + File hostIdFile = new File(destDir, "hostId"); + if (hostIdFile.exists()) { + List lines = IOUtils.readLines(new FileInputStream(hostIdFile)); + hostInstanceID = StringUtils.join(lines, "\n"); + } + + String itemStr = jsobObj.getString("Items"); + JSONObject binLogFileObj = JSONObject.parseObject(itemStr); + JSONArray items = binLogFileObj.getJSONArray("BinLogFile"); + if (items == null || items.isEmpty()) { + continue; + } + for (int i = 0; i < items.size(); i++) { + JSONObject item = (JSONObject) items.get(i); + String oneHostInstanceID = item.getString("HostInstanceID"); + if (hostInstanceID == null) { + hostInstanceID = oneHostInstanceID; + FileOutputStream hostIdFileOut = null; + try { + hostIdFileOut = new FileOutputStream(hostIdFile); + hostIdFileOut.write(oneHostInstanceID.getBytes()); + hostIdFileOut.flush(); + } finally { + IOUtils.closeQuietly(hostIdFileOut); + } + } + + if (hostInstanceID.equals(oneHostInstanceID)) { // 只选择一个host下载 + String downloadLink = item.getString("DownloadLink"); + String fileName = StringUtils.substringBetween(downloadLink, "mysql-bin.", ".tar"); + if (StringUtils.isNotEmpty(fileName)) { + File currentFile = new File(destDir, "mysql-bin." + fileName); + if (currentFile.isFile() && currentFile.exists()) { + // 检查一下文件是否存在，存在就就没必要下载了 + continue; + } + } + + HttpGet httpGet = new HttpGet(downloadLink); + CloseableHttpClient httpClient = HttpClientBuilder.create() + .setMaxConnPerRoute(50) + .setMaxConnTotal(100) + .build(); + RequestConfig requestConfig = RequestConfig.custom() + .setConnectTimeout(TIMEOUT) + .setConnectionRequestTimeout(TIMEOUT) + .setSocketTimeout(TIMEOUT) + .build(); + httpGet.setConfig(requestConfig); + HttpResponse response = httpClient.execute(httpGet); + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != HttpResponseStatus.OK.code()) { + throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + + statusCode); + } + saveFile(destDir, response); + } + } + } + } + } + + private static void saveFile(File parentFile, HttpResponse response) throws IOException { + InputStream is = response.getEntity().getContent(); + long totalSize = Long.parseLong(response.getFirstHeader("Content-Length").getValue()); + String fileName = response.getFirstHeader("Content-Disposition").getValue(); + fileName = StringUtils.substringAfter(fileName, "filename="); + boolean isTar = StringUtils.endsWith(fileName, ".tar"); + FileUtils.forceMkdir(parentFile); + FileOutputStream fos = null; + try { + if (isTar) { + TarArchiveInputStream tais = new TarArchiveInputStream(is); + TarArchiveEntry tarArchiveEntry = null; + while ((tarArchiveEntry = tais.getNextTarEntry()) != null) { + String name = tarArchiveEntry.getName(); + File tarFile = new File(parentFile, name); + logger.info("start to download file " + tarFile.getName()); + BufferedOutputStream bos = null; + try { + bos = new BufferedOutputStream(new FileOutputStream(tarFile)); + int read = -1; + byte[] buffer = new byte[1024]; + while ((read = tais.read(buffer)) != -1) { + bos.write(buffer, 0, read); + } + logger.info("download file " + tarFile.getName() + " end!"); + } finally { + IOUtils.closeQuietly(bos); + } + } + tais.close(); + } else { + File file = new File(parentFile, fileName); + if (!file.isFile()) { + file.createNewFile(); + } + try { + fos = new FileOutputStream(file); + byte[] buffer = new byte[1024]; + int len; + long copySize = 0; + long nextPrintProgress = 0; + logger.info("start to download file " + file.getName()); + while ((len = is.read(buffer)) != -1) { + fos.write(buffer, 0, len); + copySize += len; + long progress = copySize * 100 / totalSize; + if (progress >= nextPrintProgress) { + logger.info("download " + file.getName() + " progress : " + progress + + "% , download size : " + copySize + ", total size : " + totalSize); + nextPrintProgress += 10; + } + } + logger.info("download file " + file.getName() + " end!"); + fos.flush(); + } finally { + IOUtils.closeQuietly(fos); + } + } + } finally { + IOUtils.closeQuietly(fos); + } + } + + public static String describeBinlogFiles(String url, String ak, String sk, String dbInstanceId, Date startTime, + Date endTime, int pageSize, int pageNumber) throws Exception { + Map paramMap = new HashMap(); + paramMap.put("Action", "DescribeBinlogFiles"); + paramMap.put("DBInstanceId", dbInstanceId); // rds实例id + paramMap.put("StartTime", formatIso8601Date(startTime)); + paramMap.put("EndTime", formatIso8601Date(endTime)); + paramMap.put("PageSize", String.valueOf(pageSize)); + paramMap.put("PageNumber", String.valueOf(pageNumber)); + return doRequest(url, paramMap, ak, sk); + } + + private static String doRequest(String domin, Map param, String ak, String sk) throws Exception { + param.put("AccessKeyId", ak); + param.put("SignatureMethod", "HMAC-SHA1"); + param.put("SignatureVersion", SIGNATURE_VERSION); + param.put("Version", API_VERSION); + param.put("SignatureNonce", UUID.randomUUID().toString()); + param.put("Format", "JSON"); + param.put("Timestamp", formatIso8601Date(new Date())); + String signStr = generate("POST", param, sk); + param.put("Signature", signStr); + String request = concatQueryString(param); + String url = domin + "?" + request; + String result = HttpHelper.post(url, null, Collections.EMPTY_MAP, TIMEOUT); + return result; + } + + public static String concatQueryString(Map parameters) throws UnsupportedEncodingException { + if (null == parameters) { + return null; + } + StringBuilder urlBuilder = new StringBuilder(""); + for (Map.Entry entry : parameters.entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + urlBuilder.append(encode(key)); + if (val != null) { + urlBuilder.append("=").append(encode(val)); + } + urlBuilder.append("&"); + } + int strIndex = urlBuilder.length(); + if (parameters.size() > 0) { + urlBuilder.deleteCharAt(strIndex - 1); + } + return urlBuilder.toString(); + } + + public static String encode(String value) throws UnsupportedEncodingException { + return URLEncoder.encode(value, "UTF-8"); + } + + private static String formatIso8601Date(Date date) { + SimpleDateFormat df = new SimpleDateFormat(ISO8601_DATE_FORMAT); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + return df.format(date); + } + + /** + * 使用 HMAC-SHA1 签名方法对对encryptText进行签名 + * + * @param encryptText 被签名的字符串 + * @param encryptKey 密钥 + * @return + * @throws Exception + */ + public static byte[] HmacSHA1Encrypt(String encryptText, String encryptKey) throws Exception { + byte[] data = encryptKey.getBytes(ENCODING); + // 根据给定的字节数组构造一个密钥,第二参数指定一个密钥算法的名称 + SecretKey secretKey = new SecretKeySpec(data, MAC_NAME); + // 生成一个指定 Mac 算法 的 Mac 对象 + Mac mac = Mac.getInstance(MAC_NAME); + // 用给定密钥初始化 Mac 对象 + mac.init(secretKey); + + byte[] text = encryptText.getBytes(ENCODING); + // 完成 Mac 操作 + return mac.doFinal(text); + } + + private static String base64(byte input[]) throws UnsupportedEncodingException { + return new String(Base64.encodeBase64(input), ENCODING); + } + + /** 对参数名称和参数值进行URL编码 **/ + public static String generate(String method, Map parameter, String accessKeySecret) + throws Exception { + String signString = generateSignString(method, parameter); + byte[] signBytes = HmacSHA1Encrypt(signString, accessKeySecret + "&"); + String signature = base64(signBytes); + if ("POST".equals(method)) { + return signature; + } + return URLEncoder.encode(signature, "UTF-8"); + } + + private static String generateQueryString(TreeMap treeMap) { + StringBuilder canonicalizedQueryString = new StringBuilder(); + boolean first = true; + for (String key : treeMap.navigableKeySet()) { + String value = treeMap.get(key); + if (!first) { + canonicalizedQueryString.append("&"); + } + first = false; + canonicalizedQueryString.append(percentEncode(key)).append("=").append(percentEncode(value)); + } + return canonicalizedQueryString.toString(); + } + + public static String generateSignString(String httpMethod, Map parameter) throws IOException { + TreeMap sortParameter = new TreeMap(); + sortParameter.putAll(parameter); + String canonicalizedQueryString = generateQueryString(sortParameter); + if (null == httpMethod) { + throw new RuntimeException("httpMethod can not be empty"); + } + /** 构造待签名的字符串* */ + StringBuilder stringToSign = new StringBuilder(); + stringToSign.append(httpMethod).append("&"); + stringToSign.append(percentEncode("/")).append("&"); + stringToSign.append(percentEncode(canonicalizedQueryString)); + return stringToSign.toString(); + } + + public static String percentEncode(String value) { + try { + return value == null ? null : URLEncoder.encode(value, ENCODING) + .replaceAll("\\+", "%20") + .replaceAll("\\*", "%2A") + .replaceAll("%7E", "~"); + } catch (Exception e) { + } + return ""; + } +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java new file mode 100644 index 00000000..68be38e5 --- /dev/null +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/rds/RdsLocalBinlogEventParser.java @@ -0,0 +1,111 @@ +package com.alibaba.otter.canal.parse.inbound.mysql.rds; + +import java.io.File; +import java.util.Date; + +import org.apache.commons.lang.StringUtils; +import org.springframework.util.Assert; + +import com.alibaba.otter.canal.parse.CanalEventParser; +import com.alibaba.otter.canal.parse.exception.CanalParseException; +import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser; +import com.alibaba.otter.canal.protocol.position.EntryPosition; + +/** + * 基于rds binlog备份文件的复制 + * + * @author agapple 2017年10月15日 下午1:27:36 + * @since 1.0.25 + */ +public class RdsLocalBinlogEventParser extends LocalBinlogEventParser implements CanalEventParser { + + private String url = "https://rds.aliyuncs.com/"; // openapi地址 + private String accesskey; // 云账号的ak + private String secretkey; // 云账号sk + private String instanceId; // rds实例id + private Long startTime; + private Long endTime; + + public RdsLocalBinlogEventParser(){ + } + + public void start() throws CanalParseException { + try { + Assert.notNull(startTime); + Assert.notNull(accesskey); + Assert.notNull(secretkey); + Assert.notNull(instanceId); + Assert.notNull(url); + if (endTime == null) { + endTime = System.currentTimeMillis(); + } + + RdsBinlogOpenApi.downloadBinlogFiles(url, + accesskey, + secretkey, + instanceId, + new Date(startTime), + new Date(endTime), + new File(directory)); + + // 更新一下时间戳 + masterPosition = new EntryPosition(startTime); + } catch (Throwable e) { + logger.error("download binlog failed", e); + throw new CanalParseException(e); + } + + super.start(); + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + if (StringUtils.isNotEmpty(url)) { + this.url = url; + } + } + + public String getAccesskey() { + return accesskey; + } + + public void setAccesskey(String accesskey) { + this.accesskey = accesskey; + } + + public String getSecretkey() { + return secretkey; + } + + public void setSecretkey(String secretkey) { + this.secretkey = secretkey; + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Long getEndTime() { + return endTime; + } + + public void setEndTime(Long endTime) { + this.endTime = endTime; + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java similarity index 98% rename from parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java rename to parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index 4e6a392d..3c43e94a 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManager.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -37,14 +37,14 @@ import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO; import com.alibaba.otter.canal.protocol.position.EntryPosition; /** - * 基于console远程管理 see internal class: CanalTableMeta , ConsoleTableMetaTSDB + * 基于db远程管理 see internal class: CanalTableMeta , ConsoleTableMetaTSDB * * @author agapple 2017年7月27日 下午10:47:55 * @since 3.2.5 */ -public class TableMetaManager implements TableMetaTSDB { +public class DatabaseTableMeta implements TableMetaTSDB { - private static Logger logger = LoggerFactory.getLogger(TableMetaManager.class); + private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class); private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'"); private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation"); private static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L); @@ -58,7 +58,7 @@ public class TableMetaManager implements TableMetaTSDB { private MetaHistoryDAO metaHistoryDAO; private MetaSnapshotDAO metaSnapshotDAO; - public TableMetaManager(){ + public DatabaseTableMeta(){ } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java new file mode 100644 index 00000000..596847ae --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsBinlogOpenApiTest.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.parse.inbound.mysql; + +import java.io.File; +import java.util.Date; + +import org.apache.commons.lang.time.DateUtils; + +import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsBinlogOpenApi; + +/** + * @author agapple 2017年10月15日 下午2:14:34 + * @since 1.0.25 + */ +public class RdsBinlogOpenApiTest { + + public void testSimple() throws Throwable { + Date startTime = DateUtils.parseDate("2017-10-13 20:56:58", new String[] { "yyyy-MM-dd HH:mm:ss" }); + Date endTime = DateUtils.parseDate("2017-10-14 02:57:59", new String[] { "yyyy-MM-dd HH:mm:ss" }); + RdsBinlogOpenApi.downloadBinlogFiles("https://rds.aliyuncs.com/", + "", + "", + "rm-bp180v4mfjnm157es", + startTime, + endTime, + new File("/tmp/binlog/")); + } +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsLocalBinlogDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsLocalBinlogDumpTest.java new file mode 100644 index 00000000..82fe8803 --- /dev/null +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/RdsLocalBinlogDumpTest.java @@ -0,0 +1,117 @@ +package com.alibaba.otter.canal.parse.inbound.mysql; + +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.otter.canal.parse.exception.CanalParseException; +import com.alibaba.otter.canal.parse.inbound.mysql.rds.RdsLocalBinlogEventParser; +import com.alibaba.otter.canal.parse.index.AbstractLogPositionManager; +import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest; +import com.alibaba.otter.canal.parse.support.AuthenticationInfo; +import com.alibaba.otter.canal.protocol.CanalEntry.Column; +import com.alibaba.otter.canal.protocol.CanalEntry.Entry; +import com.alibaba.otter.canal.protocol.CanalEntry.EntryType; +import com.alibaba.otter.canal.protocol.CanalEntry.EventType; +import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; +import com.alibaba.otter.canal.protocol.CanalEntry.RowData; +import com.alibaba.otter.canal.protocol.position.LogPosition; +import com.alibaba.otter.canal.sink.exception.CanalSinkException; + +/** + * @author agapple 2017年10月15日 下午2:16:58 + * @since 1.0.25 + */ +public class RdsLocalBinlogDumpTest { + + @Test + public void testSimple() { + String directory = "/tmp/rds"; + final RdsLocalBinlogEventParser controller = new RdsLocalBinlogEventParser(); + controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello")); + controller.setConnectionCharset(Charset.forName("UTF-8")); + controller.setDirectory(directory); + controller.setAccesskey(""); + controller.setSecretkey(""); + controller.setInstanceId(""); + controller.setStartTime(1507860498350L); + controller.setEventSink(new AbstractCanalEventSinkTest>() { + + public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination) + throws CanalSinkException, + InterruptedException { + + for (Entry entry : entrys) { + if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN + || entry.getEntryType() == EntryType.TRANSACTIONEND) { + continue; + } + + if (entry.getEntryType() == EntryType.ROWDATA) { + RowChange rowChage = null; + try { + rowChage = RowChange.parseFrom(entry.getStoreValue()); + } catch (Exception e) { + throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + + entry.toString(), e); + } + + EventType eventType = rowChage.getEventType(); + System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", + entry.getHeader().getLogfileName(), + entry.getHeader().getLogfileOffset(), + entry.getHeader().getSchemaName(), + entry.getHeader().getTableName(), + eventType)); + + for (RowData rowData : rowChage.getRowDatasList()) { + if (eventType == EventType.DELETE) { + print(rowData.getBeforeColumnsList()); + } else if (eventType == EventType.INSERT) { + print(rowData.getAfterColumnsList()); + } else { + System.out.println("-------> before"); + print(rowData.getBeforeColumnsList()); + System.out.println("-------> after"); + print(rowData.getAfterColumnsList()); + } + } + } + } + + return true; + } + + }); + controller.setLogPositionManager(new AbstractLogPositionManager() { + + @Override + public LogPosition getLatestIndexBy(String destination) { + return null; + } + + @Override + public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException { + System.out.println(logPosition); + } + }); + + controller.start(); + + try { + Thread.sleep(100 * 1000 * 1000L); + } catch (InterruptedException e) { + Assert.fail(e.getMessage()); + } + controller.stop(); + } + + private void print(List columns) { + for (Column column : columns) { + System.out.println(column.getName() + " : " + column.getValue() + " update=" + column.getUpdated()); + } + } +} diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java index af8c9dc5..cd5bfe6e 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/TableMetaManagerTest.java @@ -26,7 +26,7 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition; public class TableMetaManagerTest { @Resource - TableMetaManager tableMetaManager; + DatabaseTableMeta tableMetaManager; @Test public void testSimple() throws FileNotFoundException, IOException { diff --git a/parse/src/test/resources/tsdb/derby-tsdb.xml b/parse/src/test/resources/tsdb/derby-tsdb.xml index a6c19486..4ce4dcbf 100644 --- a/parse/src/test/resources/tsdb/derby-tsdb.xml +++ b/parse/src/test/resources/tsdb/derby-tsdb.xml @@ -7,7 +7,7 @@ default-autowire="byName"> - + diff --git a/parse/src/test/resources/tsdb/h2-tsdb.xml b/parse/src/test/resources/tsdb/h2-tsdb.xml index 452e7ee5..c0a3d385 100644 --- a/parse/src/test/resources/tsdb/h2-tsdb.xml +++ b/parse/src/test/resources/tsdb/h2-tsdb.xml @@ -7,7 +7,7 @@ default-autowire="byName"> - + diff --git a/parse/src/test/resources/tsdb/mysql-tsdb.xml b/parse/src/test/resources/tsdb/mysql-tsdb.xml index e3743109..099c034d 100644 --- a/parse/src/test/resources/tsdb/mysql-tsdb.xml +++ b/parse/src/test/resources/tsdb/mysql-tsdb.xml @@ -7,7 +7,7 @@ default-autowire="byName"> - + diff --git a/pom.xml b/pom.xml index 649abb36..f0e4b5a5 100644 --- a/pom.xml +++ b/pom.xml @@ -173,6 +173,21 @@ commons-io 2.4 + + commons-codec + commons-codec + 1.9 + + + org.apache.commons + commons-compress + 1.9 + + + org.apache.httpcomponents + httpclient + 4.5.1 + org.apache.zookeeper zookeeper From 1463463f1e170d3e1335c8aee89ca3fbd09e1e31 Mon Sep 17 00:00:00 2001 From: agapple Date: Sun, 15 Oct 2017 16:08:29 +0800 Subject: [PATCH 06/34] fixed data --- .../resources/example/rds_instance.properties | 1 + .../inbound/mysql/tsdb/DatabaseTableMeta.java | 22 ++++++++++++++++--- .../parse/inbound/TableMetaCacheTest.java | 7 ++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/deployer/src/main/resources/example/rds_instance.properties b/deployer/src/main/resources/example/rds_instance.properties index 9712cf93..f04b3832 100644 --- a/deployer/src/main/resources/example/rds_instance.properties +++ b/deployer/src/main/resources/example/rds_instance.properties @@ -1,4 +1,5 @@ ################################################# +# rds openapi binlog canal.instance.rds.open.url=https://rds.aliyuncs.com/ canal.instance.rds.open.accesskey= canal.instance.rds.open.secretkey= diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index 3c43e94a..21edb63c 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -285,6 +285,8 @@ public class DatabaseTableMeta implements TableMetaTSDB { } private boolean compareTableMetaDbAndMemory(MysqlConnection connection, final String schema, final String table) { + TableMeta tableMetaFromMem = memoryTableMeta.find(schema, table); + TableMeta tableMetaFromDB = new TableMeta(); tableMetaFromDB.setSchema(schema); tableMetaFromDB.setTable(table); @@ -292,15 +294,29 @@ public class DatabaseTableMeta implements TableMetaTSDB { ResultSetPacket packet = connection.query("desc " + getFullName(schema, table)); tableMetaFromDB.setFields(TableMetaCache.parserTableMeta(packet)); } catch (IOException e) { + if (e.getMessage().contains("errorNumber=1146")) { + logger.error("table not exist in db , pls check :" + getFullName(schema, table) + " , mem : " + + tableMetaFromMem); + return false; + } throw new CanalParseException(e); } - TableMeta tableMetaFromMem = memoryTableMeta.find(schema, table); boolean result = compareTableMeta(tableMetaFromMem, tableMetaFromDB); if (!result) { - logger.error("compare failed . \n db : " + tableMetaFromDB + " \n mem : " + tableMetaFromMem); - } + String createDDL = null; + try { + ResultSetPacket packet = connection.query("show create table " + getFullName(schema, table)); + if (packet.getFieldValues().size() > 1) { + createDDL = packet.getFieldValues().get(1); + } + } catch (IOException e) { + // ignore + } + logger.error("pls submit github issue, show create table ddl:" + createDDL + " , compare failed . \n db : " + + tableMetaFromDB + " \n mem : " + tableMetaFromMem); + } return result; } diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java index 8a78e246..f9fc0796 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java @@ -14,17 +14,14 @@ public class TableMetaCacheTest { @Test public void testSimple() throws IOException { - - MysqlConnection connection = new MysqlConnection(new InetSocketAddress("100.81.153.8", 3308), - "jingwei", - "jingwei"); + MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"); try { connection.connect(); } catch (IOException e) { Assert.fail(e.getMessage()); } - List packets = connection.queryMulti("show create table test.ljh_test; show create table test.user"); + List packets = connection.queryMulti("show create table test.ljh_test"); String createDDL = null; if (packets.get(0).getFieldValues().size() > 0) { createDDL = packets.get(0).getFieldValues().get(1); From 8d6f19cc3300bb6a83c3628576c543293e7f6a66 Mon Sep 17 00:00:00 2001 From: agapple Date: Sun, 15 Oct 2017 16:19:02 +0800 Subject: [PATCH 07/34] fixed default value --- deployer/src/main/resources/example/instance.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties index 31b625de..22357e1f 100644 --- a/deployer/src/main/resources/example/instance.properties +++ b/deployer/src/main/resources/example/instance.properties @@ -27,7 +27,7 @@ canal.instance.dbPassword=canal canal.instance.defaultDatabaseName=test canal.instance.connectionCharset=UTF-8 # table regex -canal.instance.filter.regex=*\\..* +canal.instance.filter.regex=.*\\..* # table black regex canal.instance.filter.black.regex= ################################################# \ No newline at end of file From c56f58deae192e410bb6d63e934919078bdfa091 Mon Sep 17 00:00:00 2001 From: agapple Date: Wed, 1 Nov 2017 20:29:47 +0800 Subject: [PATCH 08/34] upgrade druid to 1.1.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0e4b5a5..5557c718 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ com.alibaba druid - 1.1.5-preview_14 + 1.1.5 From 2dc019709b661828a56b6f1111b4701d8890d527 Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 14 Nov 2017 23:31:46 +0800 Subject: [PATCH 09/34] fixed quota ddl --- .../parse/inbound/mysql/dbsync/TableMetaCache.java | 5 ++++- .../parse/inbound/mysql/ddl/DruidDdlParser.java | 12 ++++++++++++ .../parse/inbound/mysql/tsdb/MemoryTableMeta.java | 3 ++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index ff66c875..10e0e26c 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -14,6 +14,7 @@ import com.alibaba.otter.canal.parse.exception.CanalParseException; import com.alibaba.otter.canal.parse.inbound.TableMeta; import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta; import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection; +import com.alibaba.otter.canal.parse.inbound.mysql.ddl.DruidDdlParser; import com.alibaba.otter.canal.parse.inbound.mysql.tsdb.TableMetaTSDB; import com.alibaba.otter.canal.protocol.position.EntryPosition; import com.google.common.cache.CacheBuilder; @@ -103,7 +104,9 @@ public class TableMetaCache { * size), "YES")); meta.setKey("PRI".equalsIgnoreCase(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size))); - meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size)); + // 特殊处理引号 + meta.setDefaultValue(DruidDdlParser.unescapeQuotaName(packet.getFieldValues() + .get(nameMaps.get(COLUMN_DEFAULT) + i * size))); meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size)); result.add(meta); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java index b2f4fa01..958dc735 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java @@ -198,4 +198,16 @@ public class DruidDdlParser { return name; } + public static String unescapeQuotaName(String name) { + if (name.length() > 2) { + char c0 = name.charAt(0); + char x0 = name.charAt(name.length() - 1); + if (c0 == '\'' && x0 == '\'') { + return name.substring(1, name.length() - 1); + } + } + + return name; + } + } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java index 3bd220ef..5b697eb9 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -187,7 +187,7 @@ public class MemoryTableMeta implements TableMetaTSDB { if (column.getDefaultExpr() == null || column.getDefaultExpr() instanceof SQLNullExpr) { fieldMeta.setDefaultValue(null); } else { - fieldMeta.setDefaultValue(getSqlName(column.getDefaultExpr())); + fieldMeta.setDefaultValue(DruidDdlParser.unescapeQuotaName(getSqlName(column.getDefaultExpr()))); } fieldMeta.setColumnName(name); @@ -232,6 +232,7 @@ public class MemoryTableMeta implements TableMetaTSDB { return sqlName.toString(); } } + public SchemaRepository getRepository() { return repository; From a3b9f6f1ebb21dd528effcceba2ac207f40b15e8 Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 14 Nov 2017 23:44:15 +0800 Subject: [PATCH 10/34] fixed NPE --- .../otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java index 958dc735..64ddc722 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/ddl/DruidDdlParser.java @@ -187,7 +187,7 @@ public class DruidDdlParser { } public static String unescapeName(String name) { - if (name.length() > 2) { + if (name != null && name.length() > 2) { char c0 = name.charAt(0); char x0 = name.charAt(name.length() - 1); if ((c0 == '"' && x0 == '"') || (c0 == '`' && x0 == '`')) { @@ -199,7 +199,7 @@ public class DruidDdlParser { } public static String unescapeQuotaName(String name) { - if (name.length() > 2) { + if (name != null && name.length() > 2) { char c0 = name.charAt(0); char x0 = name.charAt(name.length() - 1); if (c0 == '\'' && x0 == '\'') { From 5348994b4a8e63c5ededf27adfaffcb95f1a05a6 Mon Sep 17 00:00:00 2001 From: agapple Date: Wed, 15 Nov 2017 10:42:58 +0800 Subject: [PATCH 11/34] fixed alisql heartbeat --- .../canal/parse/inbound/mysql/dbsync/LogEventConvert.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 34474efd..344fe004 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -387,6 +387,12 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0"); FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0"); tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta)); + } else if (isHeartBeat) { + // 处理alisql模式的test.heartbeat心跳数据 + // 心跳表基本无权限,需要mock一个tableMeta + FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null); + FieldMeta typeMeta = new FieldMeta("type", "int(11)", true, false, null); + tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta)); } EventType eventType = null; From 3409472eb9b8d07c0c67f4b7af4ac746edf10e69 Mon Sep 17 00:00:00 2001 From: Chuanyi Li Date: Thu, 23 Nov 2017 22:25:47 +0800 Subject: [PATCH 12/34] kill CONNECTION exception fix revert. --- .../alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 0e488162..bb8ec579 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -101,7 +101,8 @@ public class MysqlConnector { MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector); executor.update("KILL CONNECTION " + connectionId); } catch (Exception e) { - throw new IOException("KILL DUMP " + connectionId + " failure", e); + // 忽略具体异常 + logger.info("KILL DUMP " + connectionId + " failure", e); } finally { if (connector != null) { connector.disconnect(); From a9284b1b3967917ee6cef0e85787575e9b121e1e Mon Sep 17 00:00:00 2001 From: agapple Date: Mon, 4 Dec 2017 11:35:02 +0800 Subject: [PATCH 13/34] fixed compiler error --- .../canal/parse/inbound/mysql/dbsync/LogEventConvert.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java index 344fe004..27fa8117 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java @@ -364,6 +364,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar throw new TableIdNotFoundException("not found tableId:" + event.getTableId()); } + boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName()); boolean isRDSHeartBeat = tableMetaCache.isOnRDS() && isRDSHeartBeat(table.getDbName(), table.getTableName()); @@ -772,6 +773,10 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType) || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType); } + + private boolean isAliSQLHeartBeat(String schema, String table) { + return "test".equalsIgnoreCase(schema) && "heartbeat".equalsIgnoreCase(table); + } private boolean isRDSHeartBeat(String schema, String table) { return "mysql".equalsIgnoreCase(schema) && "ha_health_check".equalsIgnoreCase(table); From 01ef15223cafa277ed7f0292ce97781abaf0b73d Mon Sep 17 00:00:00 2001 From: agapple Date: Mon, 4 Dec 2017 13:19:58 +0800 Subject: [PATCH 14/34] fixed issue #415, ignore flush privileges --- .../canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java | 5 ++++- pom.xml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java index 5b697eb9..f36aa726 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/MemoryTableMeta.java @@ -64,7 +64,10 @@ public class MemoryTableMeta implements TableMetaTSDB { } try { - repository.console(ddl); + // druid暂时flush privileges语法解析有问题 + if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")) { + repository.console(ddl); + } } catch (Throwable e) { logger.warn("parse faield : " + ddl, e); } diff --git a/pom.xml b/pom.xml index 5557c718..89d68cbe 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ com.alibaba druid - 1.1.5 + 1.1.6 From 6b2d80a14590be36c5dd862804618b6568e2c3ad Mon Sep 17 00:00:00 2001 From: agapple Date: Mon, 4 Dec 2017 14:43:16 +0800 Subject: [PATCH 15/34] [maven-release-plugin] prepare release canal-1.0.25 --- client/pom.xml | 2 +- common/pom.xml | 5 ++--- dbsync/pom.xml | 2 +- deployer/pom.xml | 2 +- driver/pom.xml | 2 +- example/pom.xml | 2 +- filter/pom.xml | 2 +- instance/core/pom.xml | 2 +- instance/manager/pom.xml | 2 +- instance/pom.xml | 2 +- instance/spring/pom.xml | 2 +- meta/pom.xml | 2 +- parse/pom.xml | 5 ++--- pom.xml | 2 +- protocol/pom.xml | 2 +- server/pom.xml | 2 +- sink/pom.xml | 2 +- store/pom.xml | 2 +- 18 files changed, 20 insertions(+), 22 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 4b68d148..63ff4c55 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/common/pom.xml b/common/pom.xml index eb702d71..9dce276f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml canal.common diff --git a/dbsync/pom.xml b/dbsync/pom.xml index f3d7f62a..b00e52ac 100644 --- a/dbsync/pom.xml +++ b/dbsync/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/deployer/pom.xml b/deployer/pom.xml index 4c20ff43..5afbe140 100644 --- a/deployer/pom.xml +++ b/deployer/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/driver/pom.xml b/driver/pom.xml index 66e59ab7..8c7ec346 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/example/pom.xml b/example/pom.xml index 171f873a..e27dfae5 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/filter/pom.xml b/filter/pom.xml index ed2e5670..3213f48a 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/instance/core/pom.xml b/instance/core/pom.xml index c87de497..2cc63056 100644 --- a/instance/core/pom.xml +++ b/instance/core/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../../pom.xml canal.instance.core diff --git a/instance/manager/pom.xml b/instance/manager/pom.xml index 6c90a9cc..c307f9ba 100644 --- a/instance/manager/pom.xml +++ b/instance/manager/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../../pom.xml com.alibaba.otter diff --git a/instance/pom.xml b/instance/pom.xml index c1928e99..86fab453 100644 --- a/instance/pom.xml +++ b/instance/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/instance/spring/pom.xml b/instance/spring/pom.xml index c4012218..764e9100 100644 --- a/instance/spring/pom.xml +++ b/instance/spring/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../../pom.xml com.alibaba.otter diff --git a/meta/pom.xml b/meta/pom.xml index 06155714..4a5bdf78 100644 --- a/meta/pom.xml +++ b/meta/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/parse/pom.xml b/parse/pom.xml index f495b52c..a2a3d6bc 100644 --- a/parse/pom.xml +++ b/parse/pom.xml @@ -1,10 +1,9 @@ - + 4.0.0 com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml canal.parse diff --git a/pom.xml b/pom.xml index 89d68cbe..f2417417 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ canal pom canal module for otter ${project.version} - 1.0.25-SNAPSHOT + 1.0.25 https://github.com/alibaba/canal org.sonatype.oss diff --git a/protocol/pom.xml b/protocol/pom.xml index b13c17b1..98452404 100644 --- a/protocol/pom.xml +++ b/protocol/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/server/pom.xml b/server/pom.xml index 5538d9f5..af450b4c 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml canal.server diff --git a/sink/pom.xml b/sink/pom.xml index bffb8bdf..a75fa5da 100644 --- a/sink/pom.xml +++ b/sink/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter diff --git a/store/pom.xml b/store/pom.xml index a0b478fd..6481fbb4 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25-SNAPSHOT + 1.0.25 ../pom.xml com.alibaba.otter From 20942533e61212c48f9d0724654a1c0309665f02 Mon Sep 17 00:00:00 2001 From: agapple Date: Mon, 4 Dec 2017 14:43:29 +0800 Subject: [PATCH 16/34] [maven-release-plugin] prepare for next development iteration --- client/pom.xml | 2 +- common/pom.xml | 2 +- dbsync/pom.xml | 2 +- deployer/pom.xml | 2 +- driver/pom.xml | 2 +- example/pom.xml | 2 +- filter/pom.xml | 2 +- instance/core/pom.xml | 2 +- instance/manager/pom.xml | 2 +- instance/pom.xml | 2 +- instance/spring/pom.xml | 2 +- meta/pom.xml | 2 +- parse/pom.xml | 2 +- pom.xml | 2 +- protocol/pom.xml | 2 +- server/pom.xml | 2 +- sink/pom.xml | 2 +- store/pom.xml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 63ff4c55..2778841c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/common/pom.xml b/common/pom.xml index 9dce276f..9c57c560 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml canal.common diff --git a/dbsync/pom.xml b/dbsync/pom.xml index b00e52ac..1f274e82 100644 --- a/dbsync/pom.xml +++ b/dbsync/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/deployer/pom.xml b/deployer/pom.xml index 5afbe140..5fa2efc8 100644 --- a/deployer/pom.xml +++ b/deployer/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/driver/pom.xml b/driver/pom.xml index 8c7ec346..edd24e57 100644 --- a/driver/pom.xml +++ b/driver/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/example/pom.xml b/example/pom.xml index e27dfae5..1f60bc01 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/filter/pom.xml b/filter/pom.xml index 3213f48a..30b53e28 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/instance/core/pom.xml b/instance/core/pom.xml index 2cc63056..cbfab2fd 100644 --- a/instance/core/pom.xml +++ b/instance/core/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../../pom.xml canal.instance.core diff --git a/instance/manager/pom.xml b/instance/manager/pom.xml index c307f9ba..10f01e43 100644 --- a/instance/manager/pom.xml +++ b/instance/manager/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../../pom.xml com.alibaba.otter diff --git a/instance/pom.xml b/instance/pom.xml index 86fab453..aeec17cb 100644 --- a/instance/pom.xml +++ b/instance/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/instance/spring/pom.xml b/instance/spring/pom.xml index 764e9100..6a8f172a 100644 --- a/instance/spring/pom.xml +++ b/instance/spring/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../../pom.xml com.alibaba.otter diff --git a/meta/pom.xml b/meta/pom.xml index 4a5bdf78..6192d92d 100644 --- a/meta/pom.xml +++ b/meta/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/parse/pom.xml b/parse/pom.xml index a2a3d6bc..efdb57f2 100644 --- a/parse/pom.xml +++ b/parse/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml canal.parse diff --git a/pom.xml b/pom.xml index f2417417..c0a3c04e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ canal pom canal module for otter ${project.version} - 1.0.25 + 1.0.26-SNAPSHOT https://github.com/alibaba/canal org.sonatype.oss diff --git a/protocol/pom.xml b/protocol/pom.xml index 98452404..ec7c7867 100644 --- a/protocol/pom.xml +++ b/protocol/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/server/pom.xml b/server/pom.xml index af450b4c..1d243981 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml canal.server diff --git a/sink/pom.xml b/sink/pom.xml index a75fa5da..4ed7e04f 100644 --- a/sink/pom.xml +++ b/sink/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter diff --git a/store/pom.xml b/store/pom.xml index 6481fbb4..318c2f9d 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -3,7 +3,7 @@ com.alibaba.otter canal - 1.0.25 + 1.0.26-SNAPSHOT ../pom.xml com.alibaba.otter From 7d485efd71165d34de2ba0d721bad0c530f62310 Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 5 Dec 2017 17:01:15 +0800 Subject: [PATCH 17/34] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 702d6f4d..456bd2db 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@
  • slave重做中继日志中的事件，将改变反映它自己的数据。

    • canal的工作原理：

    -

    +

    原理相对比较简单：

    1. canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议
      2. From d1c83ea7bb81fd14051e33379a19c2351c58b8d2 Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 5 Dec 2017 19:46:49 +0800 Subject: [PATCH 18/34] fixed fastjson autotype --- .../java/com/alibaba/otter/canal/common/utils/JsonUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java index 5b42683d..97a3ded3 100644 --- a/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java @@ -10,6 +10,7 @@ import java.util.List; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; +import com.alibaba.fastjson.parser.ParserConfig; import com.alibaba.fastjson.serializer.JSONSerializer; import com.alibaba.fastjson.serializer.ObjectSerializer; import com.alibaba.fastjson.serializer.PropertyFilter; @@ -28,6 +29,7 @@ public class JsonUtils { SerializeConfig.getGlobalInstance().put(InetAddress.class, InetAddressSerializer.instance); SerializeConfig.getGlobalInstance().put(Inet4Address.class, InetAddressSerializer.instance); SerializeConfig.getGlobalInstance().put(Inet6Address.class, InetAddressSerializer.instance); + ParserConfig.getGlobalInstance().setAutoTypeSupport(true); } public static T unmarshalFromByte(byte[] bytes, Class targetClass) { From 8b57b9a0a53c46bf22e4cf683d8e616a3f560a0b Mon Sep 17 00:00:00 2001 From: agapple Date: Fri, 8 Dec 2017 22:54:22 +0800 Subject: [PATCH 19/34] fixed issue #442 , compareTableMetaDbAndMemory --- .../canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java index 21edb63c..8c57b3ac 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/DatabaseTableMeta.java @@ -248,7 +248,7 @@ public class DatabaseTableMeta implements TableMetaTSDB { boolean compareAll = true; for (Schema schema : tmpMemoryTableMeta.getRepository().getSchemas()) { for (String table : schema.showTables()) { - if (!compareTableMetaDbAndMemory(connection, schema.getName(), table)) { + if (!compareTableMetaDbAndMemory(connection, tmpMemoryTableMeta, schema.getName(), table)) { compareAll = false; } } @@ -284,7 +284,8 @@ public class DatabaseTableMeta implements TableMetaTSDB { return false; } - private boolean compareTableMetaDbAndMemory(MysqlConnection connection, final String schema, final String table) { + private boolean compareTableMetaDbAndMemory(MysqlConnection connection, MemoryTableMeta memoryTableMeta, + final String schema, final String table) { TableMeta tableMetaFromMem = memoryTableMeta.find(schema, table); TableMeta tableMetaFromDB = new TableMeta(); From 912f5db2c70eece6263e84e9f0c8316798d1955b Mon Sep 17 00:00:00 2001 From: agapple Date: Fri, 8 Dec 2017 23:20:15 +0800 Subject: [PATCH 20/34] fixed issue #440, for null position --- .../canal/parse/inbound/mysql/MysqlEventParser.java | 9 +++++++-- pom.xml | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index efee2b71..c840829b 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -385,10 +385,15 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) { // 使用一个未来极大的时间，基于位点进行定位 long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年 - return findAsPerTimestampInSpecificLogFile(mysqlConnection, + EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, startTimestamp, fixedPosition, fixedPosition.getJournalName()); + if (entryPosition == null) { + throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position" + + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition()); + } + return entryPosition; } else { return fixedPosition; } @@ -767,7 +772,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } if (StringUtils.equals(endPosition.getJournalName(), logfilename) - && endPosition.getPosition() <= logfileoffset) { + && endPosition.getPosition() < logfileoffset) { return false; } diff --git a/pom.xml b/pom.xml index c0a3c04e..a951f42c 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ com.alibaba druid - 1.1.6 + 1.1.7-preview_0 From 5aed505429694130a76115b7a4219b1a36f53346 Mon Sep 17 00:00:00 2001 From: agapple Date: Fri, 8 Dec 2017 23:21:40 +0800 Subject: [PATCH 21/34] fixed issue #440, enableTsdb --- .../parse/inbound/mysql/AbstractMysqlEventParser.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java index 6d69a836..422bae37 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java @@ -159,10 +159,11 @@ public abstract class AbstractMysqlEventParser extends AbstractEventParser { public void setTsdbSpringXml(String tsdbSpringXml) { this.tsdbSpringXml = tsdbSpringXml; - - if (tableMetaTSDB == null) { - // 初始化 - tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + if (this.enableTsdb) { + if (tableMetaTSDB == null) { + // 初始化 + tableMetaTSDB = TableMetaTSDBBuilder.build(destination, tsdbSpringXml); + } } } From c60b04424b0a357c69ced2e68a847e16ca87a002 Mon Sep 17 00:00:00 2001 From: agapple Date: Sun, 10 Dec 2017 22:24:24 +0800 Subject: [PATCH 22/34] fixed issue #440 , optime find position --- .../parse/inbound/mysql/MysqlEventParser.java | 8 ++ parse/src/test/resources/ddl/create.sql | 87 ++++++++++++++----- 2 files changed, 72 insertions(+), 23 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index c840829b..65ce2360 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -771,6 +771,14 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE } } + if (entryPosition == null) { + // 如果啥都找不到，就返回第一条Format Event的位点,一般是116位点 + entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); + logger.debug("set {} to be pending start position before finding another proper one...", + entryPosition); + logPosition.setPostion(entryPosition); + } + if (StringUtils.equals(endPosition.getJournalName(), logfilename) && endPosition.getPosition() < logfileoffset) { return false; diff --git a/parse/src/test/resources/ddl/create.sql b/parse/src/test/resources/ddl/create.sql index 6f64a3a6..74e716ab 100644 --- a/parse/src/test/resources/ddl/create.sql +++ b/parse/src/test/resources/ddl/create.sql @@ -1,23 +1,64 @@ - CREATE TABLE `test` ( - `id` bigint(20) zerofill unsigNed NOT NULL AUTO_INCREMENT COMMENT 'id', - `c_tinyint` tinyint(4) DEFAULT '1' COMMENT 'tinyint', - `c_smallint` smallint(6) DEFAULT 0 COMMENT 'smallint', - `c_mediumint` mediumint(9) DEFAULT NULL COMMENT 'mediumint', - `c_int` int(11) DEFAULT NULL COMMENT 'int', - `c_bigint` bigint(20) DEFAULT NULL COMMENT 'bigint', - `c_decimal` decimal(10,3) DEFAULT NULL COMMENT 'decimal', - `c_date` date DEFAULT '0000-00-00' COMMENT 'date', - `c_datetime` datetime DEFAULT '0000-00-00 00:00:00' COMMENT 'datetime', - `c_timestamp` timestamp NULL DEFAULT NULL COMMENT 'timestamp', - `c_time` time DEFAULT NULL COMMENT 'time', - `c_char` char(10) DEFAULT NULL COMMENT 'char', - `c_varchar` varchar(10) DEFAULT 'hello' COMMENT 'varchar', - `c_blob` blob COMMENT 'blob', - `c_text` text COMMENT 'text', - `c_mediumtext` mediumtext COMMENT 'mediumtext', - `c_longblob` longblob COMMENT 'longblob', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_a` (`c_tinyint`), - KEY `k_b` (`c_smallint`), - KEY `k_c` (`c_mediumint`,`c_int`) -) ENGINE=InnoDB AUTO_INCREMENT=1769503 DEFAULT CHARSET=utf8mb4 COMMENT='10000000'; \ No newline at end of file +CREATE TABLE `test_all` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `c_bit_1` bit(1) DEFAULT NULL, + `c_bit_8` bit(8) DEFAULT NULL, + `c_bit_16` bit(16) DEFAULT NULL, + `c_bit_32` bit(32) DEFAULT NULL, + `c_bit_64` bit(64) DEFAULT NULL, + `c_bool` boolean DEFAULT NULL, + `c_tinyint_1` tinyint(1) DEFAULT NULL, + `c_tinyint_4` tinyint(4) DEFAULT NULL, + `c_tinyint_8` tinyint(8) DEFAULT NULL, + `c_tinyint_8_un` tinyint(8) unsigned DEFAULT NULL, + `c_smallint_1` smallint(1) DEFAULT NULL, + `c_smallint_16` smallint(16) DEFAULT NULL, + `c_smallint_16_un` smallint(16) unsigned DEFAULT NULL, + `c_mediumint_1` mediumint(1) DEFAULT NULL, + `c_mediumint_24` mediumint(24) DEFAULT NULL, + `c_mediumint_24_un` mediumint(24) unsigned DEFAULT NULL, + `c_int_1` int(1) DEFAULT NULL, + `c_int_32` int(32) DEFAULT NULL, + `c_int_32_un` int(32) unsigned DEFAULT NULL, + `c_bigint_1` bigint(1) DEFAULT NULL, + `c_bigint_64` bigint(64) DEFAULT NULL, + `c_bigint_64_un` bigint(64) unsigned DEFAULT NULL, + `c_decimal` decimal(10,3) DEFAULT NULL, + `c_decimal_pr` decimal(10,3) DEFAULT NULL, + `c_float` float DEFAULT NULL, + `c_float_pr` float(10,3) DEFAULT NULL, + `c_float_un` float(10,3) unsigned DEFAULT NULL, + `c_double` double DEFAULT NULL, + `c_double_pr` double(10,3) DEFAULT NULL, + `c_double_un` double(10,3) unsigned DEFAULT NULL, + `c_date` date DEFAULT NULL COMMENT 'date', + `c_datetime` datetime DEFAULT NULL, + `c_datetime_1` datetime(1) DEFAULT NULL, + `c_datetime_3` datetime(3) DEFAULT NULL, + `c_datetime_6` datetime(6) DEFAULT NULL, + `c_timestamp` timestamp DEFAULT CURRENT_TIMESTAMP, + `c_timestamp_1` timestamp(1) DEFAULT 0, + `c_timestamp_3` timestamp(3) DEFAULT 0, + `c_timestamp_6` timestamp(6) DEFAULT 0, + `c_time` time DEFAULT NULL, + `c_time_1` time(1) DEFAULT NULL, + `c_time_3` time(3) DEFAULT NULL, + `c_time_6` time(6) DEFAULT NULL, + `c_year` year DEFAULT NULL, + `c_year_4` year(4) DEFAULT NULL, + `c_char` char(10) DEFAULT NULL, + `c_varchar` varchar(10) DEFAULT NULL, + `c_binary` binary(10) DEFAULT NULL, + `c_varbinary` varbinary(10) DEFAULT NULL, + `c_blob_tiny` tinyblob DEFAULT NULL, + `c_blob` blob DEFAULT NULL, + `c_blob_medium` mediumblob DEFAULT NULL, + `c_blob_long` longblob DEFAULT NULL, + `c_text_tiny` tinytext DEFAULT NULL, + `c_text` text DEFAULT NULL, + `c_text_medium` mediumtext DEFAULT NULL, + `c_text_long` longtext DEFAULT NULL, + `c_enum` enum('a','b','c') DEFAULT NULL, + `c_set` set('a','b','c') DEFAULT NULL, + `c_json` json DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT='10000000' \ No newline at end of file From 448fc8bcde787177db5adcc6e26bc5aae1eff4ab Mon Sep 17 00:00:00 2001 From: agapple Date: Sun, 10 Dec 2017 22:27:14 +0800 Subject: [PATCH 23/34] fixed issue #440 --- .../canal/parse/inbound/mysql/MysqlEventParser.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 65ce2360..a398ea45 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -761,14 +761,13 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE Long logposTimestamp = entry.getHeader().getExecuteTime(); Long serverId = entry.getHeader().getServerId(); - if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) - || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) { + if (logger.isDebugEnabled()) { logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] { logfilename, logfileoffset, logposTimestamp, startTimestamp }); - // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出 - if (logposTimestamp >= startTimestamp) { - return false; - } + } + // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出 + if (logposTimestamp >= startTimestamp) { + return false; } if (entryPosition == null) { From 5e206e4f7e7b62c1379f8f9b8945fd1eeb150fcf Mon Sep 17 00:00:00 2001 From: agapple Date: Sun, 10 Dec 2017 23:05:28 +0800 Subject: [PATCH 24/34] fixed issue #440 , optimize find position --- .../parse/inbound/mysql/MysqlEventParser.java | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index a398ea45..38c92a6f 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -374,7 +374,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE return findAsPerTimestampInSpecificLogFile(mysqlConnection, startTimestamp, endPosition, - endPosition.getJournalName()); + endPosition.getJournalName(), + true); } else { return endPosition; } @@ -388,7 +389,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, startTimestamp, fixedPosition, - fixedPosition.getJournalName()); + fixedPosition.getJournalName(), + true); if (entryPosition == null) { throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position" + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition()); @@ -446,7 +448,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, entryPosition.getTimestamp(), endPosition, - entryPosition.getJournalName()); + entryPosition.getJournalName(), + true); } } @@ -590,7 +593,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, startTimestamp, endPosition, - startSearchBinlogFile); + startSearchBinlogFile, + false); if (entryPosition == null) { if (StringUtils.equalsIgnoreCase(minBinlogFileName, startSearchBinlogFile)) { // 已经找到最早的一个binlog，没必要往前找了 @@ -738,7 +742,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection, final Long startTimestamp, final EntryPosition endPosition, - final String searchBinlogFile) { + final String searchBinlogFile, + final Boolean justForPositionTimestamp) { final LogPosition logPosition = new LogPosition(); try { @@ -752,6 +757,15 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE EntryPosition entryPosition = null; try { CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true); + if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) { + // 初始位点 + entryPosition = new EntryPosition(searchBinlogFile, + event.getLogPos(), + event.getWhen() * 1000, + event.getServerId()); + logPosition.setPostion(entryPosition); + } + if (entry == null) { return true; } @@ -761,21 +775,16 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE Long logposTimestamp = entry.getHeader().getExecuteTime(); Long serverId = entry.getHeader().getServerId(); - if (logger.isDebugEnabled()) { - logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] { - logfilename, logfileoffset, logposTimestamp, startTimestamp }); - } - // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出 - if (logposTimestamp >= startTimestamp) { - return false; - } - - if (entryPosition == null) { - // 如果啥都找不到，就返回第一条Format Event的位点,一般是116位点 - entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); - logger.debug("set {} to be pending start position before finding another proper one...", - entryPosition); - logPosition.setPostion(entryPosition); + if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) + || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) { + if (logger.isDebugEnabled()) { + logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] { + logfilename, logfileoffset, logposTimestamp, startTimestamp }); + } + // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出 + if (logposTimestamp >= startTimestamp) { + return false; + } } if (StringUtils.equals(endPosition.getJournalName(), logfilename) @@ -788,14 +797,18 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // data.length，代表该事务的下一条offest，避免多余的事务重复 if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) { entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); - logger.debug("set {} to be pending start position before finding another proper one...", - entryPosition); + if (logger.isDebugEnabled()) { + logger.debug("set {} to be pending start position before finding another proper one...", + entryPosition); + } logPosition.setPostion(entryPosition); } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) { // 当前事务开始位点 entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId); - logger.debug("set {} to be pending start position before finding another proper one...", - entryPosition); + if (logger.isDebugEnabled()) { + logger.debug("set {} to be pending start position before finding another proper one...", + entryPosition); + } logPosition.setPostion(entryPosition); } From 8b03f172ed987f723898c90a0b77c6188be4f874 Mon Sep 17 00:00:00 2001 From: zhengmingzhi Date: Tue, 12 Dec 2017 19:17:53 +0800 Subject: [PATCH 25/34] =?UTF-8?q?=E8=A7=A3=E5=86=B3destination=E6=97=A0?= =?UTF-8?q?=E9=99=90=E8=BF=9E=E6=8E=A5=E7=AD=89=E5=BE=85=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../parse/driver/mysql/MysqlConnector.java | 8 +++-- .../driver/mysql/socket/SocketChannel.java | 30 +++++++++++++++++++ .../driver/mysql/utils/PacketManager.java | 10 +++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 0e488162..645066e1 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -43,6 +43,8 @@ public class MysqlConnector { // mysql connectinnId private long connectionId = -1; private AtomicBoolean connected = new AtomicBoolean(false); + + public static final int timeout = 3000; // 3s public MysqlConnector(){ } @@ -144,8 +146,8 @@ public class MysqlConnector { } private void negotiate(SocketChannel channel) throws IOException { - HeaderPacket header = PacketManager.readHeader(channel, 4); - byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength()); + HeaderPacket header = PacketManager.readHeader(channel, 4, timeout); + byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout); if (body[0] < 0) {// check field_count if (body[0] == -1) { ErrorPacket error = new ErrorPacket(); @@ -184,7 +186,7 @@ public class MysqlConnector { header = null; header = PacketManager.readHeader(channel, 4); body = null; - body = PacketManager.readBytes(channel, header.getPacketBodyLength()); + body = PacketManager.readBytes(channel, header.getPacketBodyLength(), timeout); assert body != null; if (body[0] < 0) { if (body[0] == -1) { diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java index a4568887..40ce6c32 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java @@ -64,6 +64,36 @@ public class SocketChannel { } } while (true); } + + public byte[] read(int readSize, int timeout) throws IOException { + int accumulatedWaitTime = 0; + do { + if (readSize > cache.readableBytes()) { + if (null == channel) { + throw new IOException("socket has Interrupted !"); + } + + accumulatedWaitTime += 100; + if (accumulatedWaitTime > timeout) { + throw new IOException("socket read timeout occured !"); + } + + synchronized (this) { + try { + wait(100); + } catch (InterruptedException e) { + throw new IOException("socket has Interrupted !"); + } + } + } else { + byte[] back = new byte[readSize]; + synchronized (lock) { + cache.readBytes(back); + } + return back; + } + } while (true); + } public boolean isConnected() { return channel != null ? true : false; diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java index 38f9f103..7d4b9020 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java @@ -13,9 +13,19 @@ public abstract class PacketManager { return header; } + public static HeaderPacket readHeader(SocketChannel ch, int len, int timeout) throws IOException { + HeaderPacket header = new HeaderPacket(); + header.fromBytes(ch.read(len, timeout)); + return header; + } + public static byte[] readBytes(SocketChannel ch, int len) throws IOException { return ch.read(len); } + + public static byte[] readBytes(SocketChannel ch, int len, int timeout) throws IOException { + return ch.read(len, timeout); + } public static void writePkg(SocketChannel ch, byte[]... srcs) throws IOException { ch.writeChannel(srcs); From d66839c965d9070ee00061619fff97e55e0bbaf4 Mon Sep 17 00:00:00 2001 From: lucifax301 <1918831771@qq.com> Date: Mon, 18 Dec 2017 14:10:53 +0800 Subject: [PATCH 26/34] mysql semi support and mariadb gtid parse --- .../taobao/tddl/dbsync/binlog/LogBuffer.java | 1 + .../taobao/tddl/dbsync/binlog/LogDecoder.java | 3 + .../taobao/tddl/dbsync/binlog/LogEvent.java | 17 +++++- .../event/mariadb/MariaGtidLogEvent.java | 26 +++++++++ .../packets/client/SemiAckCommandPacket.java | 58 +++++++++++++++++++ .../parse/driver/mysql/utils/ByteHelper.java | 13 ++++- .../parse/inbound/mysql/MysqlConnection.java | 28 +++++++++ .../mysql/dbsync/DirectLogFetcher.java | 22 ++++++- 8 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java index f81660fa..5557bea7 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java @@ -20,6 +20,7 @@ public class LogBuffer { protected int origin, limit; protected int position; + protected int semival; protected LogBuffer(){ } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java index 95a57424..0f9c32b5 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java @@ -104,6 +104,9 @@ public final class LogDecoder { try { /* Decoding binary-log to event */ event = decode(buffer, header, context); + if(event!=null){ + event.setSemival(buffer.semival); + } } catch (IOException e) { if (logger.isWarnEnabled()) logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: " + context.getLogPosition(), e); diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java index 4dba80ee..d8abf4c1 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java @@ -359,8 +359,23 @@ public abstract class LogEvent { protected static final Log logger = LogFactory.getLog(LogEvent.class); protected final LogHeader header; + + /** + * mysql半同步semi标识 0不需要semi ack 给mysql, 1需要semi ack给mysql + */ + protected int semival; + + - protected LogEvent(LogHeader header){ + public int getSemival() { + return semival; + } + + public void setSemival(int semival) { + this.semival = semival; + } + + protected LogEvent(LogHeader header){ this.header = header; } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java index 9dc22b39..6b00a633 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java @@ -13,9 +13,35 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader; */ public class MariaGtidLogEvent extends IgnorableLogEvent { + private long gtid; + + /** + * mariadb gtidlog event format + * + uint<8> GTID sequence + uint<4> Replication Domain ID + uint<1> Flags + + if flag & FL_GROUP_COMMIT_ID + + uint<8> commit_id + + else + + uint<6> 0 + + * + */ + public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ super(header, buffer, descriptionEvent); + gtid=buffer.getUlong64().longValue(); // do nothing , just ignore log event } + public long getGtid() { + return gtid; + } + + } diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java new file mode 100644 index 00000000..f96238b3 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.client; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.apache.commons.lang.StringUtils; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +/** + * semi ack command + * @author amos_chen + * + */ +public class SemiAckCommandPacket extends CommandPacket { + + public long binlogPosition; + + public String binlogFileName; + + public SemiAckCommandPacket(){ + + } + + @Override + public void fromBytes(byte[] data) throws IOException { + // TODO Auto-generated method stub + + } + + /** + * 
      +     * Bytes                        Name
+     *  --------------------------------------------------------
+     *  Bytes                        Name
+     *  -----                        ----
+     *  1                            semi mark
+     *  8                            binlog position to start at (little endian)
+     *  n                            binlog file name 
+     * 
+     *
      + */ + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + //0 write semi mark + out.write(0xef); + //1 write 8 bytes for position + ByteHelper.write8ByteUnsignedIntLittleEndian(binlogPosition, out); + + //2 write binlog filename + if(StringUtils.isNotEmpty(binlogFileName)){ + out.write(binlogFileName.getBytes()); + } + return out.toByteArray(); + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java index 2c7cd31b..67c3830e 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java @@ -108,7 +108,18 @@ public abstract class ByteHelper { return out.toByteArray(); } - + + public static void write8ByteUnsignedIntLittleEndian(long data, ByteArrayOutputStream out) { + out.write((byte) (data & 0xFF)); + out.write((byte) (data >>> 8)); + out.write((byte) (data >>> 16)); + out.write((byte) (data >>> 24)); + out.write((byte) (data >>> 32)); + out.write((byte) (data >>> 40)); + out.write((byte) (data >>> 48)); + out.write((byte) (data >>> 56)); + } + public static void writeUnsignedIntLittleEndian(long data, ByteArrayOutputStream out) { out.write((byte) (data & 0xFF)); out.write((byte) (data >>> 8)); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 7b141010..ff9c3718 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -14,6 +14,7 @@ import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor; import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor; import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.SemiAckCommandPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; import com.alibaba.otter.canal.parse.exception.CanalParseException; @@ -24,6 +25,7 @@ import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; public class MysqlConnection implements ErosaConnection { @@ -35,6 +37,8 @@ public class MysqlConnection implements ErosaConnection { private BinlogFormat binlogFormat; private BinlogImage binlogImage; + private String binlogfilename; + // tsdb releated private AuthenticationInfo authInfo; protected int connTimeout = 5 * 1000; // 5秒 @@ -141,9 +145,18 @@ public class MysqlConnection implements ErosaConnection { if (event == null) { throw new CanalParseException("parse failed"); } + + //binlog日志文件发生变化 + if(event.getHeader().getType()==LogEvent.ROTATE_EVENT){ + binlogfilename=((RotateLogEvent)event).getFilename(); + } if (!func.sink(event)) { break; + }else{ + if(event.getSemival()==1){ + sendSemiAck(binlogfilename, binlogPosition); + } } } } @@ -167,6 +180,21 @@ public class MysqlConnection implements ErosaConnection { PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody); connector.setDumping(true); } + + private void sendSemiAck(String binlogfilename, Long binlogPosition) throws IOException { + SemiAckCommandPacket semiAckCmd = new SemiAckCommandPacket(); + semiAckCmd.binlogFileName = binlogfilename; + semiAckCmd.binlogPosition = binlogPosition; + + byte[] cmdBody = semiAckCmd.toBytes(); + + logger.info("SEMI ACK with position:{}", semiAckCmd); + HeaderPacket semiAckHeader = new HeaderPacket(); + semiAckHeader.setPacketBodyLength(cmdBody.length); + semiAckHeader.setPacketSequenceNumber((byte) 0x00); + PacketManager.writePkg(connector.getChannel(), semiAckHeader.toBytes(), cmdBody); + + } public MysqlConnection fork() { MysqlConnection connection = new MysqlConnection(); diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java index ad0e95e7..8827a87a 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java @@ -36,6 +36,8 @@ public class DirectLogFetcher extends LogFetcher { public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1); private SocketChannel channel; + + private boolean issemi=false; // private BufferedInputStream input; @@ -53,6 +55,10 @@ public class DirectLogFetcher extends LogFetcher { public void start(SocketChannel channel) throws IOException { this.channel = channel; + String dbsemi= System.getProperty("db.semi"); + if("1".equals(dbsemi)){ + issemi=true; + } // 和mysql driver一样，提供buffer机制，提升读取binlog速度 // this.input = new // BufferedInputStream(channel.socket().getInputStream(), 16384); @@ -105,7 +111,15 @@ public class DirectLogFetcher extends LogFetcher { + ", len = " + netlen); } } - + + //if mysql is in semi mode + if(issemi){ + //parse semi mark + int semimark=getUint8(NET_HEADER_SIZE+1); + int semival=getUint8(NET_HEADER_SIZE+2); + this.semival=semival; + } + // The first packet is a multi-packet, concatenate the packets. while (netlen == MAX_PACKET_LENGTH) { if (!fetch0(0, NET_HEADER_SIZE)) { @@ -122,7 +136,11 @@ public class DirectLogFetcher extends LogFetcher { } // Preparing buffer variables to decoding. - origin = NET_HEADER_SIZE + 1; + if(issemi){ + origin = NET_HEADER_SIZE + 3; + }else{ + origin = NET_HEADER_SIZE + 1; + } position = origin; limit -= origin; return true; From 9595e6abbd0cc9551ba079326ce32e8f8691c398 Mon Sep 17 00:00:00 2001 From: agapple Date: Wed, 27 Dec 2017 16:53:09 +0800 Subject: [PATCH 27/34] fixed semi sync format & refactor --- .../taobao/tddl/dbsync/binlog/LogBuffer.java | 2 +- .../taobao/tddl/dbsync/binlog/LogDecoder.java | 10 ++-- .../taobao/tddl/dbsync/binlog/LogEvent.java | 9 +++- .../event/mariadb/MariaGtidLogEvent.java | 41 +++++++------- .../packets/client/SemiAckCommandPacket.java | 53 +++++++++---------- .../parse/inbound/mysql/MysqlConnection.java | 22 +++----- .../mysql/dbsync/DirectLogFetcher.java | 34 ++++++------ 7 files changed, 81 insertions(+), 90 deletions(-) diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java index 5557bea7..be61c99f 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java @@ -20,7 +20,7 @@ public class LogBuffer { protected int origin, limit; protected int position; - protected int semival; + protected int semival; protected LogBuffer(){ } diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java index 0f9c32b5..34cb70c5 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java @@ -104,12 +104,14 @@ public final class LogDecoder { try { /* Decoding binary-log to event */ event = decode(buffer, header, context); - if(event!=null){ - event.setSemival(buffer.semival); + if (event != null) { + event.setSemival(buffer.semival); } } catch (IOException e) { - if (logger.isWarnEnabled()) logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) - + " failed from: " + context.getLogPosition(), e); + if (logger.isWarnEnabled()) { + logger.warn("Decoding " + LogEvent.getTypeName(header.getType()) + " failed from: " + + context.getLogPosition(), e); + } throw e; } finally { buffer.limit(limit); /* Restore limit */ diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java index d8abf4c1..806b34f7 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java @@ -361,9 +361,14 @@ public abstract class LogEvent { protected final LogHeader header; /** - * mysql半同步semi标识 0不需要semi ack 给mysql, 1需要semi ack给mysql + * mysql半同步semi标识 + * + * 
      +     * 0不需要semi ack 给mysql
+     * 1需要semi ack给mysql
+     *
      */ - protected int semival; + protected int semival; diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java index 6b00a633..7c5ed4e5 100644 --- a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java @@ -13,35 +13,30 @@ import com.taobao.tddl.dbsync.binlog.event.LogHeader; */ public class MariaGtidLogEvent extends IgnorableLogEvent { - private long gtid; - - /** - * mariadb gtidlog event format - * - uint<8> GTID sequence - uint<4> Replication Domain ID - uint<1> Flags + private long gtid; - if flag & FL_GROUP_COMMIT_ID - - uint<8> commit_id - - else - - uint<6> 0 + /** + * 
      +     * mariadb gtidlog event format
+     *     uint<8> GTID sequence
+     *     uint<4> Replication Domain ID
+     *     uint<1> Flags
+     * 
+     * 	if flag & FL_GROUP_COMMIT_ID
+     * 	    uint<8> commit_id
+     * 	else
+     * 	    uint<6> 0
+     *
      + */ - * - */ - public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ super(header, buffer, descriptionEvent); - gtid=buffer.getUlong64().longValue(); + gtid = buffer.getUlong64().longValue(); // do nothing , just ignore log event } - public long getGtid() { - return gtid; - } + public long getGtid() { + return gtid; + } - } diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java index f96238b3..8041aecf 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/SemiAckCommandPacket.java @@ -10,26 +10,23 @@ import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; /** * semi ack command + * * @author amos_chen - * */ public class SemiAckCommandPacket extends CommandPacket { - public long binlogPosition; - - public String binlogFileName; - - public SemiAckCommandPacket(){ - - } - - @Override - public void fromBytes(byte[] data) throws IOException { - // TODO Auto-generated method stub + public long binlogPosition; + public String binlogFileName; - } + public SemiAckCommandPacket(){ - /** + } + + @Override + public void fromBytes(byte[] data) throws IOException { + } + + /** * 
            * Bytes                        Name
      *  --------------------------------------------------------
@@ -37,22 +34,22 @@ public class SemiAckCommandPacket extends CommandPacket {
      *  -----                        ----
      *  1                            semi mark
      *  8                            binlog position to start at (little endian)
-     *  n                            binlog file name 
+     *  n                            binlog file name
      * 
      *
      */ - public byte[] toBytes() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - //0 write semi mark - out.write(0xef); - //1 write 8 bytes for position - ByteHelper.write8ByteUnsignedIntLittleEndian(binlogPosition, out); - - //2 write binlog filename - if(StringUtils.isNotEmpty(binlogFileName)){ - out.write(binlogFileName.getBytes()); - } - return out.toByteArray(); - } + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + // 0 write semi mark + out.write(0xef); + // 1 write 8 bytes for position + ByteHelper.write8ByteUnsignedIntLittleEndian(binlogPosition, out); + + // 2 write binlog filename + if (StringUtils.isNotEmpty(binlogFileName)) { + out.write(binlogFileName.getBytes()); + } + return out.toByteArray(); + } } diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index ff9c3718..239e2c09 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -25,7 +25,6 @@ import com.alibaba.otter.canal.parse.support.AuthenticationInfo; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; -import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; public class MysqlConnection implements ErosaConnection { @@ -37,8 +36,6 @@ public class MysqlConnection implements ErosaConnection { private BinlogFormat binlogFormat; private BinlogImage binlogImage; - private String binlogfilename; - // tsdb releated private AuthenticationInfo authInfo; protected int connTimeout = 5 * 1000; // 5秒 @@ -145,18 +142,13 @@ public class MysqlConnection implements ErosaConnection { if (event == null) { throw new CanalParseException("parse failed"); } - - //binlog日志文件发生变化 - if(event.getHeader().getType()==LogEvent.ROTATE_EVENT){ - binlogfilename=((RotateLogEvent)event).getFilename(); - } if (!func.sink(event)) { break; - }else{ - if(event.getSemival()==1){ - sendSemiAck(binlogfilename, binlogPosition); - } + } + + if (event.getSemival() == 1) { + sendSemiAck(context.getLogPosition().getFileName(), binlogPosition); } } } @@ -180,12 +172,12 @@ public class MysqlConnection implements ErosaConnection { PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody); connector.setDumping(true); } - + private void sendSemiAck(String binlogfilename, Long binlogPosition) throws IOException { SemiAckCommandPacket semiAckCmd = new SemiAckCommandPacket(); semiAckCmd.binlogFileName = binlogfilename; semiAckCmd.binlogPosition = binlogPosition; - + byte[] cmdBody = semiAckCmd.toBytes(); logger.info("SEMI ACK with position:{}", semiAckCmd); @@ -193,7 +185,7 @@ public class MysqlConnection implements ErosaConnection { semiAckHeader.setPacketBodyLength(cmdBody.length); semiAckHeader.setPacketSequenceNumber((byte) 0x00); PacketManager.writePkg(connector.getChannel(), semiAckHeader.toBytes(), cmdBody); - + } public MysqlConnection fork() { diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java index 8827a87a..1e8d70df 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java @@ -36,8 +36,8 @@ public class DirectLogFetcher extends LogFetcher { public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1); private SocketChannel channel; - - private boolean issemi=false; + + private boolean issemi = false; // private BufferedInputStream input; @@ -55,9 +55,9 @@ public class DirectLogFetcher extends LogFetcher { public void start(SocketChannel channel) throws IOException { this.channel = channel; - String dbsemi= System.getProperty("db.semi"); - if("1".equals(dbsemi)){ - issemi=true; + String dbsemi = System.getProperty("db.semi"); + if ("1".equals(dbsemi)) { + issemi = true; } // 和mysql driver一样，提供buffer机制，提升读取binlog速度 // this.input = new @@ -111,15 +111,15 @@ public class DirectLogFetcher extends LogFetcher { + ", len = " + netlen); } } - - //if mysql is in semi mode - if(issemi){ - //parse semi mark - int semimark=getUint8(NET_HEADER_SIZE+1); - int semival=getUint8(NET_HEADER_SIZE+2); - this.semival=semival; + + // if mysql is in semi mode + if (issemi) { + // parse semi mark + int semimark = getUint8(NET_HEADER_SIZE + 1); + int semival = getUint8(NET_HEADER_SIZE + 2); + this.semival = semival; } - + // The first packet is a multi-packet, concatenate the packets. while (netlen == MAX_PACKET_LENGTH) { if (!fetch0(0, NET_HEADER_SIZE)) { @@ -136,10 +136,10 @@ public class DirectLogFetcher extends LogFetcher { } // Preparing buffer variables to decoding. - if(issemi){ - origin = NET_HEADER_SIZE + 3; - }else{ - origin = NET_HEADER_SIZE + 1; + if (issemi) { + origin = NET_HEADER_SIZE + 3; + } else { + origin = NET_HEADER_SIZE + 1; } position = origin; limit -= origin; From 3104b4cc24bcb53214ec1e33d148950a29db9160 Mon Sep 17 00:00:00 2001 From: agapple Date: Wed, 27 Dec 2017 17:34:46 +0800 Subject: [PATCH 28/34] fixed issue #449 , optimize findTransactionBeginPosition --- .../parse/inbound/mysql/MysqlEventParser.java | 79 +++++-------------- 1 file changed, 21 insertions(+), 58 deletions(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java index 38c92a6f..49ade7d9 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java @@ -8,7 +8,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimerTask; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang.StringUtils; @@ -500,10 +499,10 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE // 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据 private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException { - // 尝试找到一个合适的位置 - final AtomicBoolean reDump = new AtomicBoolean(false); + // 针对开始的第一条为非Begin记录，需要从该binlog扫描 + final AtomicLong preTransactionStartPosition = new AtomicLong(0L); mysqlConnection.reconnect(); - mysqlConnection.seek(entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction() { + mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction() { private LogPosition lastPosition; @@ -514,69 +513,33 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE return true; } - // 直接查询第一条业务数据，确认是否为事务Begin/End - if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType() - || CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) { - lastPosition = buildLastPosition(entry); - return false; - } else { - reDump.set(true); - lastPosition = buildLastPosition(entry); - return false; + // 直接查询第一条业务数据，确认是否为事务Begin + // 记录一下transaction begin position + if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN + && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) { + preTransactionStartPosition.set(entry.getHeader().getLogfileOffset()); } + + if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) { + return false;// 退出 + } + + lastPosition = buildLastPosition(entry); } catch (Exception e) { - // 上一次记录的poistion可能为一条update/insert/delete变更事件，直接进行dump的话，会缺少tableMap事件，导致tableId未进行解析 processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition()); - reDump.set(true); return false; } + + return running; } }); - // 针对开始的第一条为非Begin记录，需要从该binlog扫描 - if (reDump.get()) { - final AtomicLong preTransactionStartPosition = new AtomicLong(0L); - mysqlConnection.reconnect(); - mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction() { - private LogPosition lastPosition; - - public boolean sink(LogEvent event) { - try { - CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true); - if (entry == null) { - return true; - } - - // 直接查询第一条业务数据，确认是否为事务Begin - // 记录一下transaction begin position - if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN - && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) { - preTransactionStartPosition.set(entry.getHeader().getLogfileOffset()); - } - - if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) { - return false;// 退出 - } - - lastPosition = buildLastPosition(entry); - } catch (Exception e) { - processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition()); - return false; - } - - return running; - } - }); - - // 判断一下找到的最接近position的事务头的位置 - if (preTransactionStartPosition.get() > entryPosition.getPosition()) { - logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data"); - throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data"); - } - return preTransactionStartPosition.get(); - } else { - return entryPosition.getPosition(); + // 判断一下找到的最接近position的事务头的位置 + if (preTransactionStartPosition.get() > entryPosition.getPosition()) { + logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data"); + throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data"); } + return preTransactionStartPosition.get(); } // 根据时间查找binlog位置 From 0c7aed197a5a4f33cdab13121791e2772b99db70 Mon Sep 17 00:00:00 2001 From: jason huang Date: Tue, 2 Jan 2018 23:29:15 +0800 Subject: [PATCH 29/34] cannot build mysql connection. when invoke connect(), it will return a ChannelFuture instance, and then the "addListener" method is invoked by biz thread, however, "channelRead" method in BusinessHandler is executed by netty I/O thread. ChannelFutureListener -> operationComplete() cannot guarantee that which is executed before than BusinessHandler -> channelRead() method. reproduce it : set below jvm parameter: -Dio.netty.eventLoopThreads=1 It will caused new connection is timeout when AbstractEventParser -> start() -> parseThread ->erosaConnection.reconnect() is invoked. --- .../mysql/socket/SocketChannelPool.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java index 52609b19..53b78f06 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java @@ -54,31 +54,25 @@ public abstract class SocketChannelPool { } public static SocketChannel open(SocketAddress address) throws Exception { - final SocketChannel socket = new SocketChannel(); - final BooleanMutex mutex = new BooleanMutex(false); - boot.connect(address).addListener(new ChannelFutureListener() { + SocketChannel socket = null; + ChannelFuture future = boot.connect(address).sync(); - @Override - public void operationComplete(ChannelFuture arg0) throws Exception { - if (arg0.isSuccess()) { - socket.setChannel(arg0.channel()); - } + if (future.isSuccess()) { + future.channel().pipeline().get(BusinessHandler.class).latch.await(); + socket = chManager.get(future.channel()); + } - mutex.set(true); - } - }); - // wait for complete - mutex.get(); - if (null == socket.getChannel()) { + if (null == socket) { throw new IOException("can't create socket!"); } - chManager.put(socket.getChannel(), socket); + return socket; } public static class BusinessHandler extends ChannelInboundHandlerAdapter { private SocketChannel socket = null; + private final CountDownLatch latch = new CountDownLatch(1); @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { @@ -86,11 +80,22 @@ public abstract class SocketChannelPool { chManager.remove(ctx.channel());// 移除 } + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + socket = new SocketChannel(); + socket.setChannel(ctx.channel()); + chManager.put(ctx.channel(), socket); + latch.countDown(); + super.channelActive(ctx); + } + @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (null == socket) socket = chManager.get(ctx.channel()); if (socket != null) { socket.writeCache((ByteBuf) msg); + } else { + //TODO: need graceful error handler. + logger.error("no socket available."); } ReferenceCountUtil.release(msg);// 添加防止内存泄漏的 } From af289cd9fbbb56f6fb0efc7ae2ff32171d59dffb Mon Sep 17 00:00:00 2001 From: agapple Date: Fri, 5 Jan 2018 19:09:46 +0800 Subject: [PATCH 30/34] fixed compiler error --- .../mysql/socket/SocketChannelPool.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java index 53b78f06..79a2f691 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java @@ -6,7 +6,6 @@ import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.AdaptiveRecvByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelInitializer; @@ -20,8 +19,10 @@ import java.io.IOException; import java.net.SocketAddress; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; -import com.alibaba.otter.canal.common.utils.BooleanMutex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author luoyaogui 实现channel的管理（监听连接、读数据、回收） 2016-12-28 @@ -29,9 +30,10 @@ import com.alibaba.otter.canal.common.utils.BooleanMutex; @SuppressWarnings({ "rawtypes", "deprecation" }) public abstract class SocketChannelPool { - private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组 - private static Bootstrap boot = new Bootstrap(); // 主 + private static EventLoopGroup group = new NioEventLoopGroup(); // 非阻塞IO线程组 + private static Bootstrap boot = new Bootstrap(); // 主 private static Map chManager = new ConcurrentHashMap(); + private static final Logger logger = LoggerFactory.getLogger(SocketChannelPool.class); static { boot.group(group) @@ -71,8 +73,8 @@ public abstract class SocketChannelPool { public static class BusinessHandler extends ChannelInboundHandlerAdapter { - private SocketChannel socket = null; - private final CountDownLatch latch = new CountDownLatch(1); + private SocketChannel socket = null; + private final CountDownLatch latch = new CountDownLatch(1); @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { @@ -81,20 +83,20 @@ public abstract class SocketChannelPool { } @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - socket = new SocketChannel(); - socket.setChannel(ctx.channel()); - chManager.put(ctx.channel(), socket); - latch.countDown(); - super.channelActive(ctx); - } + public void channelActive(ChannelHandlerContext ctx) throws Exception { + socket = new SocketChannel(); + socket.setChannel(ctx.channel()); + chManager.put(ctx.channel(), socket); + latch.countDown(); + super.channelActive(ctx); + } @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { if (socket != null) { socket.writeCache((ByteBuf) msg); } else { - //TODO: need graceful error handler. + // TODO: need graceful error handler. logger.error("no socket available."); } ReferenceCountUtil.release(msg);// 添加防止内存泄漏的 From 93be9d1909d231b7176292fb565e5963c157fd57 Mon Sep 17 00:00:00 2001 From: jason huang Date: Tue, 9 Jan 2018 18:21:14 +0800 Subject: [PATCH 31/34] dump connection disconnected dump connection will be disconnected since SocketChannel.cache is overflow. --- .../parse/driver/mysql/socket/SocketChannelPool.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java index 79a2f691..1b7db29f 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannelPool.java @@ -7,7 +7,7 @@ import io.netty.channel.AdaptiveRecvByteBufAllocator; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.EventLoopGroup; @@ -71,7 +71,7 @@ public abstract class SocketChannelPool { return socket; } - public static class BusinessHandler extends ChannelInboundHandlerAdapter { + public static class BusinessHandler extends SimpleChannelInboundHandler { private SocketChannel socket = null; private final CountDownLatch latch = new CountDownLatch(1); @@ -92,18 +92,19 @@ public abstract class SocketChannelPool { } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) throws Exception { if (socket != null) { - socket.writeCache((ByteBuf) msg); + socket.writeCache(msg); } else { // TODO: need graceful error handler. logger.error("no socket available."); } - ReferenceCountUtil.release(msg);// 添加防止内存泄漏的 } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + //need output error for troubeshooting. + logger.error("business error.", cause); ctx.close(); } } From a965d2cd04b49f07b1694e83223437a0d723acc3 Mon Sep 17 00:00:00 2001 From: jason huang Date: Tue, 9 Jan 2018 18:25:51 +0800 Subject: [PATCH 32/34] dump connection is disconnected dump connection is disconnected since SocketChannel.cache is overflow. --- .../driver/mysql/socket/SocketChannel.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java index 40ce6c32..9eee2ef7 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/socket/SocketChannel.java @@ -27,10 +27,22 @@ public class SocketChannel { this.channel = channel; } - public void writeCache(ByteBuf buf) { + public void writeCache(ByteBuf buf) throws InterruptedException { synchronized (lock) { - cache.discardReadBytes();// 回收内存 - cache.writeBytes(buf); + while (true) { + cache.discardReadBytes();// 回收内存 + //source buffer is empty. + if (!buf.isReadable()) { + break; + } + + if (cache.isWritable()) { + cache.writeBytes(buf, Math.min(cache.writableBytes(), buf.readableBytes())); + } else { + //dest buffer is full. + lock.wait(100); + } + } } } From 9316a3df6d4e885e8b0344d7a90e07308e48c123 Mon Sep 17 00:00:00 2001 From: agapple Date: Wed, 10 Jan 2018 13:49:25 +0800 Subject: [PATCH 33/34] fixed issue #482 , retry for getTableMetaByDB --- .../otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java index 10e0e26c..cc1e5797 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java @@ -53,7 +53,7 @@ public class TableMetaCache { public TableMeta load(String name) throws Exception { try { return getTableMetaByDB(name); - } catch (CanalParseException e) { + } catch (Throwable e) { // 尝试做一次retry操作 try { connection.reconnect(); From 91ac0e30aa5ff41937a05077ace692847fae3e46 Mon Sep 17 00:00:00 2001 From: agapple Date: Wed, 10 Jan 2018 14:35:26 +0800 Subject: [PATCH 34/34] fixed issue #483 , support show slave hosts --- .../parse/driver/mysql/MysqlConnector.java | 5 + .../client/RegisterSlaveCommandPacket.java | 57 ++++++++++ .../parse/inbound/mysql/MysqlConnection.java | 35 +++++- .../canal/parse/DirectLogFetcherTest.java | 102 +++++++++++++++++- 4 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/RegisterSlaveCommandPacket.java diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java index 1b77a1fd..db83f241 100644 --- a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -336,4 +336,9 @@ public class MysqlConnector { public void setConnTimeout(int connTimeout) { this.connTimeout = connTimeout; } + + public String getPassword() { + return password; + } + } diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/RegisterSlaveCommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/RegisterSlaveCommandPacket.java new file mode 100644 index 00000000..c11a1fe0 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/RegisterSlaveCommandPacket.java @@ -0,0 +1,57 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.client; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +/** + * COM_REGISTER_SLAVE + * + * @author zhibinliu + * @since 1.0.24 + */ +public class RegisterSlaveCommandPacket extends CommandPacket { + + public String reportHost; + public int reportPort; + public String reportUser; + public String reportPasswd; + public long serverId; + + public RegisterSlaveCommandPacket(){ + setCommand((byte) 0x15); + } + + public void fromBytes(byte[] data) { + // bypass + } + + public static byte[] toLH(int n) { + byte[] b = new byte[4]; + b[0] = (byte) (n & 0xff); + b[1] = (byte) (n >> 8 & 0xff); + b[2] = (byte) (n >> 16 & 0xff); + b[3] = (byte) (n >> 24 & 0xff); + return b; + } + + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(getCommand()); + ByteHelper.writeUnsignedIntLittleEndian(serverId, out); + out.write((byte) reportHost.getBytes().length); + ByteHelper.writeFixedLengthBytesFromStart(reportHost.getBytes(), reportHost.getBytes().length, out); + out.write((byte) reportUser.getBytes().length); + ByteHelper.writeFixedLengthBytesFromStart(reportUser.getBytes(), reportUser.getBytes().length, out); + out.write((byte) reportPasswd.getBytes().length); + ByteHelper.writeFixedLengthBytesFromStart(reportPasswd.getBytes(), reportPasswd.getBytes().length, out); + ByteHelper.writeUnsignedShortLittleEndian(reportPort, out); + ByteHelper.writeUnsignedIntLittleEndian(0, out);// Fake + // rpl_recovery_rank + ByteHelper.writeUnsignedIntLittleEndian(0, out);// master id + return out.toByteArray(); + } + +} diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java index 239e2c09..8e9e734b 100644 --- a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java +++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java @@ -14,7 +14,9 @@ import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor; import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor; import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.SemiAckCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; import com.alibaba.otter.canal.parse.exception.CanalParseException; @@ -130,6 +132,7 @@ public class MysqlConnection implements ErosaConnection { public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException { updateSettings(); + sendRegisterSlave(); sendBinlogDump(binlogfilename, binlogPosition); DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize()); fetcher.start(connector.getChannel()); @@ -157,11 +160,40 @@ public class MysqlConnection implements ErosaConnection { throw new NullPointerException("Not implement yet"); } + private void sendRegisterSlave() throws IOException { + RegisterSlaveCommandPacket cmd = new RegisterSlaveCommandPacket(); + cmd.reportHost = authInfo.getAddress().getAddress().getHostAddress(); + cmd.reportPasswd = authInfo.getPassword(); + cmd.reportUser = authInfo.getUsername(); + cmd.reportPort = authInfo.getAddress().getPort(); // 暂时先用master节点的port + cmd.serverId = this.slaveId; + byte[] cmdBody = cmd.toBytes(); + + logger.info("Register slave {}", cmd); + + HeaderPacket header = new HeaderPacket(); + header.setPacketBodyLength(cmdBody.length); + header.setPacketSequenceNumber((byte) 0x00); + PacketManager.writePkg(connector.getChannel(), header.toBytes(), cmdBody); + + header = PacketManager.readHeader(connector.getChannel(), 4); + byte[] body = PacketManager.readBytes(connector.getChannel(), header.getPacketBodyLength()); + assert body != null; + if (body[0] < 0) { + if (body[0] == -1) { + ErrorPacket err = new ErrorPacket(); + err.fromBytes(body); + throw new IOException("Error When doing Register slave:" + err.toString()); + } else { + throw new IOException("unpexpected packet with field_count=" + body[0]); + } + } + } + private void sendBinlogDump(String binlogfilename, Long binlogPosition) throws IOException { BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket(); binlogDumpCmd.binlogFileName = binlogfilename; binlogDumpCmd.binlogPosition = binlogPosition; - // binlogDumpCmd.slaveServerId = this.slaveId; binlogDumpCmd.slaveServerId = this.slaveId; byte[] cmdBody = binlogDumpCmd.toBytes(); @@ -185,7 +217,6 @@ public class MysqlConnection implements ErosaConnection { semiAckHeader.setPacketBodyLength(cmdBody.length); semiAckHeader.setPacketSequenceNumber((byte) 0x00); PacketManager.writePkg(connector.getChannel(), semiAckHeader.toBytes(), cmdBody); - } public MysqlConnection fork() { diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java index 707eaa20..aa48f9a3 100644 --- a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java +++ b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java @@ -3,27 +3,38 @@ package com.alibaba.otter.canal.parse; import java.io.IOException; import java.net.InetSocketAddress; +import org.apache.commons.lang.StringUtils; import org.junit.Assert; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector; +import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor; import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher; import com.taobao.tddl.dbsync.binlog.LogContext; import com.taobao.tddl.dbsync.binlog.LogDecoder; import com.taobao.tddl.dbsync.binlog.LogEvent; +import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; public class DirectLogFetcherTest { + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + @Test public void testSimple() { DirectLogFetcher fetcher = new DirectLogFetcher(); try { - MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"); + MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxx", "xxxx"); connector.connect(); - sendBinlogDump(connector, "mysql-bin.001016", 4L, 3); + updateSettings(connector); + sendRegisterSlave(connector, 3); + sendBinlogDump(connector, "mysql-bin.000001", 4L, 3); fetcher.start(connector.getChannel()); @@ -42,6 +53,7 @@ public class DirectLogFetcherTest { case LogEvent.ROTATE_EVENT: // binlogFileName = ((RotateLogEvent) // event).getFilename(); + System.out.println(((RotateLogEvent) event).getFilename()); break; case LogEvent.WRITE_ROWS_EVENT_V1: case LogEvent.WRITE_ROWS_EVENT: @@ -82,6 +94,33 @@ public class DirectLogFetcherTest { } + private void sendRegisterSlave(MysqlConnector connector, int slaveId) throws IOException { + RegisterSlaveCommandPacket cmd = new RegisterSlaveCommandPacket(); + cmd.reportHost = connector.getAddress().getAddress().getHostAddress(); + cmd.reportPasswd = connector.getPassword(); + cmd.reportUser = connector.getUsername(); + cmd.serverId = slaveId; + byte[] cmdBody = cmd.toBytes(); + + HeaderPacket header = new HeaderPacket(); + header.setPacketBodyLength(cmdBody.length); + header.setPacketSequenceNumber((byte) 0x00); + PacketManager.writePkg(connector.getChannel(), header.toBytes(), cmdBody); + + header = PacketManager.readHeader(connector.getChannel(), 4); + byte[] body = PacketManager.readBytes(connector.getChannel(), header.getPacketBodyLength()); + assert body != null; + if (body[0] < 0) { + if (body[0] == -1) { + ErrorPacket err = new ErrorPacket(); + err.fromBytes(body); + throw new IOException("Error When doing Register slave:" + err.toString()); + } else { + throw new IOException("unpexpected packet with field_count=" + body[0]); + } + } + } + private void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId) throws IOException { BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket(); @@ -95,4 +134,63 @@ public class DirectLogFetcherTest { binlogDumpHeader.setPacketSequenceNumber((byte) 0x00); PacketManager.writePkg(connector.getChannel(), binlogDumpHeader.toBytes(), cmdBody); } + + private void updateSettings(MysqlConnector connector) throws IOException { + try { + update("set wait_timeout=9999999", connector); + } catch (Exception e) { + logger.warn("update wait_timeout failed", e); + } + try { + update("set net_write_timeout=1800", connector); + } catch (Exception e) { + logger.warn("update net_write_timeout failed", e); + } + + try { + update("set net_read_timeout=1800", connector); + } catch (Exception e) { + logger.warn("update net_read_timeout failed", e); + } + + try { + // 设置服务端返回结果时不做编码转化，直接按照数据库的二进制编码进行发送，由客户端自己根据需求进行编码转化 + update("set names 'binary'", connector); + } catch (Exception e) { + logger.warn("update names failed", e); + } + + try { + // mysql5.6针对checksum支持需要设置session变量 + // 如果不设置会出现错误： Slave can not handle replication events with the + // checksum that master is configured to log + // 但也不能乱设置，需要和mysql server的checksum配置一致，不然RotateLogEvent会出现乱码 + update("set @master_binlog_checksum= '@@global.binlog_checksum'", connector); + } catch (Exception e) { + logger.warn("update master_binlog_checksum failed", e); + } + + try { + // 参考:https://github.com/alibaba/canal/issues/284 + // mysql5.6需要设置slave_uuid避免被server kill链接 + update("set @slave_uuid=uuid()", connector); + } catch (Exception e) { + if (!StringUtils.contains(e.getMessage(), "Unknown system variable")) { + logger.warn("update slave_uuid failed", e); + } + } + + try { + // mariadb针对特殊的类型，需要设置session变量 + update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'", connector); + } catch (Exception e) { + logger.warn("update mariadb_slave_capability failed", e); + } + } + + public void update(String cmd, MysqlConnector connector) throws IOException { + MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector); + exector.update(cmd); + } + }