From a0defe575bf505d7b91edff443f3fbe0d0fe75af Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 23 Sep 2014 15:11:35 +0800 Subject: [PATCH] init --- .gitignore | 15 + LICENSE.txt | 202 + README.md | 71 + RELEASE.txt | 3 + client/pom.xml | 101 + .../otter/canal/client/CanalConnector.java | 156 + .../otter/canal/client/CanalConnectors.java | 70 + .../canal/client/CanalNodeAccessStrategy.java | 14 + .../client/impl/ClusterCanalConnector.java | 327 + .../impl/ClusterNodeAccessStrategy.java | 112 + .../client/impl/SimpleCanalConnector.java | 475 + .../client/impl/SimpleNodeAccessStrategy.java | 35 + .../impl/running/ClientRunningData.java | 39 + .../impl/running/ClientRunningListener.java | 23 + .../impl/running/ClientRunningMonitor.java | 228 + .../canal/client/running/AbstractZkTest.java | 18 + .../client/running/ClientRunningTest.java | 136 + client/src/test/java/logback.xml | 14 + common/pom.xml | 69 + .../canal/common/AbstractCanalLifeCycle.java | 33 + .../otter/canal/common/CanalException.java | 37 + .../otter/canal/common/CanalLifeCycle.java | 14 + .../canal/common/alarm/CanalAlarmHandler.java | 21 + .../canal/common/alarm/LogAlarmHandler.java | 22 + .../canal/common/utils/AddressUtils.java | 95 + .../canal/common/utils/BooleanMutex.java | 157 + .../common/utils/CanalToStringStyle.java | 80 + .../otter/canal/common/utils/JsonUtils.java | 73 + .../common/utils/NamedThreadFactory.java | 56 + .../otter/canal/common/utils/UriUtils.java | 79 + .../common/zookeeper/ByteSerializer.java | 32 + .../common/zookeeper/StringSerializer.java | 32 + .../canal/common/zookeeper/ZkClientx.java | 146 + .../canal/common/zookeeper/ZooKeeperx.java | 181 + .../common/zookeeper/ZookeeperPathUtils.java | 181 + .../zookeeper/running/ServerRunningData.java | 59 + .../running/ServerRunningListener.java | 31 + .../running/ServerRunningMonitor.java | 273 + .../running/ServerRunningMonitors.java | 36 + .../otter/canal/common/AbstractZkTest.java | 18 + .../otter/canal/common/ServerRunningTest.java | 143 + common/src/test/java/logback.xml | 14 + dbsync/pom.xml | 43 + .../tddl/dbsync/binlog/CharsetConversion.java | 367 + .../tddl/dbsync/binlog/DirectLogFetcher.java | 487 ++ .../tddl/dbsync/binlog/FileLogFetcher.java | 160 + .../taobao/tddl/dbsync/binlog/LogBuffer.java | 1995 +++++ .../taobao/tddl/dbsync/binlog/LogContext.java | 77 + .../taobao/tddl/dbsync/binlog/LogDecoder.java | 494 ++ .../taobao/tddl/dbsync/binlog/LogEvent.java | 437 + .../taobao/tddl/dbsync/binlog/LogFetcher.java | 93 + .../tddl/dbsync/binlog/LogPosition.java | 127 + .../binlog/event/AppendBlockLogEvent.java | 53 + .../binlog/event/BeginLoadQueryLogEvent.java | 20 + .../binlog/event/CreateFileLogEvent.java | 73 + .../binlog/event/DeleteFileLogEvent.java | 33 + .../binlog/event/DeleteRowsLogEvent.java | 19 + .../binlog/event/ExecuteLoadLogEvent.java | 33 + .../event/ExecuteLoadQueryLogEvent.java | 101 + .../event/FormatDescriptionLogEvent.java | 305 + .../dbsync/binlog/event/GtidLogEvent.java | 45 + .../binlog/event/HeartbeatLogEvent.java | 49 + .../binlog/event/IgnorableLogEvent.java | 35 + .../dbsync/binlog/event/IncidentLogEvent.java | 88 + .../dbsync/binlog/event/IntvarLogEvent.java | 98 + .../dbsync/binlog/event/LoadLogEvent.java | 391 + .../tddl/dbsync/binlog/event/LogHeader.java | 310 + .../binlog/event/PreviousGtidsLogEvent.java | 19 + .../dbsync/binlog/event/QueryLogEvent.java | 920 ++ .../dbsync/binlog/event/RandLogEvent.java | 83 + .../dbsync/binlog/event/RotateLogEvent.java | 145 + .../dbsync/binlog/event/RowsLogBuffer.java | 976 +++ .../dbsync/binlog/event/RowsLogEvent.java | 221 + .../binlog/event/RowsQueryLogEvent.java | 33 + .../dbsync/binlog/event/StartLogEventV3.java | 61 + .../dbsync/binlog/event/StopLogEvent.java | 22 + .../dbsync/binlog/event/TableMapLogEvent.java | 543 ++ .../dbsync/binlog/event/UnknownLogEvent.java | 17 + .../binlog/event/UpdateRowsLogEvent.java | 22 + .../dbsync/binlog/event/UserVarLogEvent.java | 146 + .../binlog/event/WriteRowsLogEvent.java | 19 + .../tddl/dbsync/binlog/event/XidLogEvent.java | 32 + .../event/mariadb/AnnotateRowsEvent.java | 33 + .../mariadb/BinlogCheckPointLogEvent.java | 21 + .../event/mariadb/MariaGtidListLogEvent.java | 21 + .../event/mariadb/MariaGtidLogEvent.java | 21 + .../dbsync/binlog/BaseLogFetcherTest.java | 109 + .../dbsync/binlog/DirectLogFetcherTest.java | 90 + .../dbsync/binlog/FileLogFetcherTest.java | 93 + .../tddl/dbsync/binlog/LogBufferTest.java | 356 + .../test/resources/binlog/mysql-bin.000001 | Bin 0 -> 22920 bytes dbsync/src/test/resources/dummy.txt | 1 + deployer/pom.xml | 118 + deployer/src/main/assembly/dev.xml | 54 + deployer/src/main/assembly/release.xml | 54 + deployer/src/main/bin/startup.bat | 25 + deployer/src/main/bin/startup.sh | 104 + deployer/src/main/bin/stop.sh | 53 + .../otter/canal/deployer/CanalConstants.java | 50 + .../otter/canal/deployer/CanalController.java | 452 + .../otter/canal/deployer/CanalLauncher.java | 58 + .../otter/canal/deployer/InstanceConfig.java | 93 + .../deployer/monitor/InstanceAction.java | 25 + .../monitor/InstanceConfigMonitor.java | 16 + .../monitor/ManagerInstanceConfigMonitor.java | 20 + .../monitor/SpringInstanceConfigMonitor.java | 297 + deployer/src/main/resources/canal.properties | 62 + .../resources/example/instance.properties | 27 + deployer/src/main/resources/logback.xml | 83 + .../resources/spring/default-instance.xml | 183 + .../main/resources/spring/file-instance.xml | 168 + .../main/resources/spring/group-instance.xml | 253 + .../main/resources/spring/memory-instance.xml | 156 + driver/pom.xml | 47 + .../parse/driver/mysql/MysqlConnector.java | 294 + .../driver/mysql/MysqlQueryExecutor.java | 107 + .../driver/mysql/MysqlUpdateExecutor.java | 56 + .../driver/mysql/packets/CommandPacket.java | 24 + .../driver/mysql/packets/HeaderPacket.java | 73 + .../parse/driver/mysql/packets/IPacket.java | 29 + .../mysql/packets/PacketWithHeaderPacket.java | 32 + .../client/BinlogDumpCommandPacket.java | 70 + .../client/ClientAuthenticationPacket.java | 130 + .../packets/client/QueryCommandPacket.java | 34 + .../mysql/packets/server/DataPacket.java | 17 + .../mysql/packets/server/EOFPacket.java | 46 + .../mysql/packets/server/ErrorPacket.java | 66 + .../mysql/packets/server/FieldPacket.java | 190 + .../server/HandshakeInitializationPacket.java | 91 + .../driver/mysql/packets/server/OKPacket.java | 118 + .../mysql/packets/server/Reply323Packet.java | 27 + .../packets/server/ResultSetHeaderPacket.java | 67 + .../mysql/packets/server/ResultSetPacket.java | 42 + .../mysql/packets/server/RowDataPacket.java | 38 + .../mysql/utils/BinlogDumpCommandBuilder.java | 38 + .../parse/driver/mysql/utils/ByteHelper.java | 158 + .../mysql/utils/ChannelBufferHelper.java | 62 + .../mysql/utils/LengthCodedStringReader.java | 43 + .../canal/parse/driver/mysql/utils/MSC.java | 26 + .../mysql/utils/MySQLPasswordEncrypter.java | 74 + .../driver/mysql/utils/PacketManager.java | 67 + .../driver/mysql/MysqlConnectorTest.java | 54 + example/pom.xml | 123 + example/src/main/assembly/dev.xml | 54 + example/src/main/assembly/release.xml | 54 + example/src/main/bin/startup.bat | 26 + example/src/main/bin/startup.sh | 96 + example/src/main/bin/stop.sh | 53 + example/src/main/conf/logback.xml | 73 + .../example/AbstractCanalClientTest.java | 253 + .../canal/example/ClusterCanalClientTest.java | 52 + .../canal/example/SimpleCanalClientTest.java | 48 + example/src/main/resources/logback.xml | 25 + filter/pom.xml | 39 + .../otter/canal/filter/CanalEventFilter.java | 13 + .../otter/canal/filter/PatternUtils.java | 48 + .../canal/filter/aviater/AviaterELFilter.java | 37 + .../filter/aviater/AviaterRegexFilter.java | 128 + .../filter/aviater/AviaterSimpleFilter.java | 53 + .../canal/filter/aviater/RegexFunction.java | 32 + .../exception/CanalFilterException.java | 35 + .../otter/canal/filter/AviaterFilterTest.java | 108 + .../canal/filter/MutliAviaterFilterTest.java | 63 + instance/core/pom.xml | 35 + .../canal/instance/core/CanalInstance.java | 35 + .../instance/core/CanalInstanceGenerator.java | 16 + .../instance/core/CanalInstanceSupport.java | 105 + instance/manager/pom.xml | 20 + .../instance/manager/CanalConfigClient.java | 29 + .../manager/CanalInstanceWithManager.java | 550 ++ .../ManagerCanalInstanceGenerator.java | 29 + .../canal/instance/manager/model/Canal.java | 88 + .../manager/model/CanalParameter.java | 865 ++ .../instance/manager/model/CanalStatus.java | 22 + instance/pom.xml | 19 + instance/spring/pom.xml | 26 + .../spring/CanalInstanceWithSpring.java | 181 + .../spring/SpringCanalInstanceGenerator.java | 32 + .../PropertyPlaceholderConfigurer.java | 167 + .../spring/support/SocketAddressEditor.java | 28 + .../integrated/DefaultSpringInstanceTest.java | 48 + .../integrated/GroupSpringInstanceTest.java | 48 + .../integrated/MemorySpringInstanceTest.java | 48 + .../src/test/resources/canal.properties | 54 + .../test/resources/retl/instance.properties | 37 + .../resources/spring/default-instance.xml | 173 + .../test/resources/spring/file-instance.xml | 159 + .../test/resources/spring/group-instance.xml | 236 + .../test/resources/spring/memory-instance.xml | 147 + meta/pom.xml | 31 + .../otter/canal/meta/CanalMetaManager.java | 93 + .../canal/meta/FileMixedMetaManager.java | 375 + .../otter/canal/meta/MemoryMetaManager.java | 226 + .../otter/canal/meta/MixedMetaManager.java | 185 + .../canal/meta/PeriodMixedMetaManager.java | 168 + .../canal/meta/ZooKeeperMetaManager.java | 329 + .../exception/CanalMetaManagerException.java | 33 + .../canal/meta/AbstractMetaManagerTest.java | 117 + .../otter/canal/meta/AbstractZkTest.java | 18 + .../canal/meta/FileMixedMetaManagerTest.java | 88 + .../canal/meta/MemoryMetaManagerTest.java | 40 + .../canal/meta/MixedMetaManagerTest.java | 104 + .../meta/PeriodMixedMetaManagerTest.java | 94 + .../canal/meta/ZooKeeperMetaManagerTest.java | 62 + parse/pom.xml | 56 + .../otter/canal/parse/CanalEventParser.java | 13 + .../otter/canal/parse/CanalHASwitchable.java | 16 + .../parse/exception/CanalHAException.java | 35 + .../parse/exception/CanalParseException.java | 35 + .../exception/TableIdNotFoundException.java | 29 + .../canal/parse/ha/CanalHAController.java | 17 + .../canal/parse/ha/HeartBeatHAController.java | 63 + .../parse/inbound/AbstractBinlogParser.java | 25 + .../parse/inbound/AbstractEventParser.java | 519 ++ .../canal/parse/inbound/BinlogParser.java | 17 + .../canal/parse/inbound/ErosaConnection.java | 30 + .../parse/inbound/EventTransactionBuffer.java | 164 + .../parse/inbound/HeartBeatCallback.java | 21 + .../canal/parse/inbound/SinkFunction.java | 12 + .../otter/canal/parse/inbound/TableMeta.java | 122 + .../parse/inbound/group/GroupEventParser.java | 57 + .../mysql/AbstractMysqlEventParser.java | 94 + .../inbound/mysql/DbsyncMysqlEventParser.java | 5 + .../inbound/mysql/LocalBinLogConnection.java | 244 + .../inbound/mysql/LocalBinlogEventParser.java | 118 + .../parse/inbound/mysql/MysqlConnection.java | 306 + .../parse/inbound/mysql/MysqlEventParser.java | 738 ++ .../inbound/mysql/SlaveEntryPosition.java | 31 + .../mysql/dbsync/DirectLogFetcher.java | 180 + .../inbound/mysql/dbsync/LogEventConvert.java | 707 ++ .../inbound/mysql/dbsync/SimpleDdlParser.java | 275 + .../inbound/mysql/dbsync/TableMetaCache.java | 139 + .../inbound/mysql/local/BinLogFileQueue.java | 225 + .../mysql/local/BufferedFileDataInput.java | 92 + .../parse/index/CanalLogPositionManager.java | 18 + .../index/FailbackLogPositionManager.java | 74 + .../index/FileMixedLogPositionManager.java | 194 + .../parse/index/MemoryLogPositionManager.java | 39 + .../parse/index/MetaLogPositionManager.java | 73 + .../parse/index/MixedLogPositionManager.java | 90 + .../index/PeriodMixedLogPositionManager.java | 111 + .../index/ZooKeeperLogPositionManager.java | 57 + .../parse/support/AuthenticationInfo.java | 127 + .../parse/support/HaAuthenticationInfo.java | 36 + .../canal/parse/DirectLogFetcherTest.java | 101 + .../canal/parse/helper/TimeoutChecker.java | 59 + .../inbound/EventTransactionBufferTest.java | 138 + .../parse/inbound/TableMetaCacheTest.java | 33 + .../parse/inbound/group/DummyEventStore.java | 158 + .../inbound/group/GroupEventPaserTest.java | 103 + .../inbound/mysql/LocalBinlogDumpTest.java | 111 + .../mysql/LocalBinlogEventParserTest.java | 238 + .../parse/inbound/mysql/MysqlDumpTest.java | 114 + .../inbound/mysql/MysqlEventParserTest.java | 308 + .../inbound/mysql/SimpleDdlParserTest.java | 170 + .../index/AbstractLogPositionManagerTest.java | 38 + .../canal/parse/index/AbstractZkTest.java | 18 + .../FileMixedLogPositionManagerTest.java | 49 + .../index/MemoryLogPositionManagerTest.java | 14 + .../index/MetaLogPositionManagerTest.java | 88 + .../index/MixedLogPositionManagerTest.java | 52 + .../PeriodMixedLogPositionManagerTest.java | 52 + .../ZooKeeperLogPositionManagerTest.java | 35 + .../stub/AbstractCanalEventSinkTest.java | 11 + .../stub/AbstractCanalLogPositionManager.java | 8 + .../test/resources/binlog/mysql-bin.000001 | Bin 0 -> 22920 bytes .../test/resources/binlog/mysql-bin.000002 | Bin 0 -> 302 bytes parse/src/test/resources/dummy.txt | 1 + pom.xml | 407 + protocol/pom.xml | 30 + .../otter/canal/protocol/CanalEntry.java | 7734 +++++++++++++++++ .../otter/canal/protocol/CanalPacket.java | 7073 +++++++++++++++ .../otter/canal/protocol/CanalProtocol.proto | 114 + .../otter/canal/protocol/ClientIdentity.java | 102 + .../otter/canal/protocol/EntryProtocol.proto | 196 + .../alibaba/otter/canal/protocol/Message.java | 56 + .../exception/CanalClientException.java | 32 + .../protocol/position/EntryPosition.java | 109 + .../canal/protocol/position/LogIdentity.java | 72 + .../canal/protocol/position/LogPosition.java | 69 + .../protocol/position/MetaqPosition.java | 46 + .../canal/protocol/position/Position.java | 20 + .../protocol/position/PositionRange.java | 106 + .../canal/protocol/position/TimePosition.java | 56 + server/pom.xml | 37 + .../otter/canal/server/CanalServer.java | 17 + .../embeded/CanalServerWithEmbeded.java | 459 + .../exception/CanalServerException.java | 35 + .../server/netty/CanalServerWithNetty.java | 105 + .../otter/canal/server/netty/NettyUtils.java | 55 + .../handler/ClientAuthenticationHandler.java | 121 + .../handler/FixedHeaderFrameDecoder.java | 21 + .../HandshakeInitializationHandler.java | 30 + .../server/netty/handler/SessionHandler.java | 249 + .../BaseCanalServerWithEmbededTest.java | 197 + ...CanalServerWithEmbeded_StandaloneTest.java | 56 + .../CanalServerWithEmbeded_StandbyTest.java | 68 + .../server/CanalServerWithNettyTest.java | 263 + sink/pom.xml | 41 + .../AbstractCanalEventDownStreamHandler.java | 25 + .../canal/sink/AbstractCanalEventSink.java | 53 + .../sink/CanalEventDownStreamHandler.java | 27 + .../otter/canal/sink/CanalEventSink.java | 38 + .../canal/sink/entry/EntryEventSink.java | 206 + .../entry/HeartBeatEntryEventHandler.java | 41 + .../canal/sink/entry/group/GroupBarrier.java | 42 + .../sink/entry/group/GroupEventSink.java | 75 + .../sink/entry/group/TimelineBarrier.java | 145 + .../group/TimelineTransactionBarrier.java | 118 + .../sink/exception/CanalSinkException.java | 35 + .../otter/canal/sink/GroupEventSinkTest.java | 117 + .../canal/sink/stub/DummyEventStore.java | 93 + store/pom.xml | 36 + .../canal/store/AbstractCanalGroupStore.java | 29 + .../store/AbstractCanalStoreScavenge.java | 116 + .../otter/canal/store/CanalEventStore.java | 84 + .../canal/store/CanalGroupEventStore.java | 12 + .../canal/store/CanalStoreConstants.java | 21 + .../canal/store/CanalStoreException.java | 35 + .../otter/canal/store/CanalStoreScavenge.java | 22 + .../alibaba/otter/canal/store/StoreInfo.java | 28 + .../canal/store/helper/CanalEventUtils.java | 93 + .../memory/MemoryEventStoreWithBuffer.java | 566 ++ .../otter/canal/store/model/BatchMode.java | 24 + .../otter/canal/store/model/Event.java | 50 + .../otter/canal/store/model/Events.java | 44 + .../memory/buffer/MemoryEventStoreBase.java | 49 + .../buffer/MemoryEventStoreMemBatchTest.java | 317 + .../MemoryEventStoreMultiThreadTest.java | 191 + .../buffer/MemoryEventStorePutAndGetTest.java | 177 + .../MemoryEventStoreRollbackAndAckTest.java | 141 + 331 files changed, 53031 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 RELEASE.txt create mode 100644 client/pom.xml create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningData.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningListener.java create mode 100644 client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java create mode 100644 client/src/test/java/com/alibaba/otter/canal/client/running/AbstractZkTest.java create mode 100644 client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java create mode 100644 client/src/test/java/logback.xml create mode 100644 common/pom.xml create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/AbstractCanalLifeCycle.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/CanalException.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/alarm/LogAlarmHandler.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/CanalToStringStyle.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ByteSerializer.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/StringSerializer.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningData.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningListener.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java create mode 100644 common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java create mode 100644 common/src/test/java/com/alibaba/otter/canal/common/AbstractZkTest.java create mode 100644 common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java create mode 100644 common/src/test/java/logback.xml create mode 100644 dbsync/pom.xml create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java create mode 100644 dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java create mode 100644 dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java create mode 100644 dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java create mode 100644 dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java create mode 100644 dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java create mode 100644 dbsync/src/test/resources/binlog/mysql-bin.000001 create mode 100644 dbsync/src/test/resources/dummy.txt create mode 100644 deployer/pom.xml create mode 100644 deployer/src/main/assembly/dev.xml create mode 100644 deployer/src/main/assembly/release.xml create mode 100755 deployer/src/main/bin/startup.bat create mode 100644 deployer/src/main/bin/startup.sh create mode 100644 deployer/src/main/bin/stop.sh create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/InstanceConfig.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java create mode 100644 deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java create mode 100644 deployer/src/main/resources/canal.properties create mode 100644 deployer/src/main/resources/example/instance.properties create mode 100644 deployer/src/main/resources/logback.xml create mode 100644 deployer/src/main/resources/spring/default-instance.xml create mode 100644 deployer/src/main/resources/spring/file-instance.xml create mode 100644 deployer/src/main/resources/spring/group-instance.xml create mode 100644 deployer/src/main/resources/spring/memory-instance.xml create mode 100644 driver/pom.xml create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/CommandPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/PacketWithHeaderPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QueryCommandPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/DataPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/EOFPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ErrorPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/HandshakeInitializationPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/OKPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/Reply323Packet.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetHeaderPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/BinlogDumpCommandBuilder.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ChannelBufferHelper.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/LengthCodedStringReader.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MSC.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MySQLPasswordEncrypter.java create mode 100644 driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java create mode 100644 driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java create mode 100644 example/pom.xml create mode 100644 example/src/main/assembly/dev.xml create mode 100644 example/src/main/assembly/release.xml create mode 100755 example/src/main/bin/startup.bat create mode 100644 example/src/main/bin/startup.sh create mode 100644 example/src/main/bin/stop.sh create mode 100644 example/src/main/conf/logback.xml create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/ClusterCanalClientTest.java create mode 100644 example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java create mode 100644 example/src/main/resources/logback.xml create mode 100644 filter/pom.xml create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/CanalEventFilter.java create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java create mode 100644 filter/src/main/java/com/alibaba/otter/canal/filter/exception/CanalFilterException.java create mode 100644 filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java create mode 100644 filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java create mode 100644 instance/core/pom.xml create mode 100644 instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java create mode 100644 instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceGenerator.java create mode 100644 instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceSupport.java create mode 100644 instance/manager/pom.xml create mode 100644 instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalConfigClient.java create mode 100644 instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java create mode 100644 instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/ManagerCanalInstanceGenerator.java create mode 100644 instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/Canal.java create mode 100644 instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java create mode 100644 instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalStatus.java create mode 100644 instance/pom.xml create mode 100644 instance/spring/pom.xml create mode 100644 instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java create mode 100644 instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java create mode 100644 instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java create mode 100644 instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java create mode 100644 instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/DefaultSpringInstanceTest.java create mode 100644 instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/GroupSpringInstanceTest.java create mode 100644 instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/MemorySpringInstanceTest.java create mode 100644 instance/spring/src/test/resources/canal.properties create mode 100644 instance/spring/src/test/resources/retl/instance.properties create mode 100644 instance/spring/src/test/resources/spring/default-instance.xml create mode 100644 instance/spring/src/test/resources/spring/file-instance.xml create mode 100644 instance/spring/src/test/resources/spring/group-instance.xml create mode 100644 instance/spring/src/test/resources/spring/memory-instance.xml create mode 100644 meta/pom.xml create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/CanalMetaManager.java create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java create mode 100644 meta/src/main/java/com/alibaba/otter/canal/meta/exception/CanalMetaManagerException.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/AbstractZkTest.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/MixedMetaManagerTest.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManagerTest.java create mode 100644 meta/src/test/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManagerTest.java create mode 100644 parse/pom.xml create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/CanalEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/CanalHASwitchable.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalHAException.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalParseException.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/exception/TableIdNotFoundException.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/ha/CanalHAController.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/ha/HeartBeatHAController.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractBinlogParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/HeartBeatCallback.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/SinkFunction.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/DbsyncMysqlEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/SlaveEntryPosition.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BufferedFileDataInput.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/FailbackLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManager.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java create mode 100644 parse/src/main/java/com/alibaba/otter/canal/parse/support/HaAuthenticationInfo.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java create mode 100644 parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java create mode 100644 parse/src/test/resources/binlog/mysql-bin.000001 create mode 100644 parse/src/test/resources/binlog/mysql-bin.000002 create mode 100644 parse/src/test/resources/dummy.txt create mode 100644 pom.xml create mode 100644 protocol/pom.xml create mode 100755 protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java create mode 100755 protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalProtocol.proto create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/EntryProtocol.proto create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/exception/CanalClientException.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogIdentity.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogPosition.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/MetaqPosition.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/Position.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/PositionRange.java create mode 100644 protocol/src/main/java/com/alibaba/otter/canal/protocol/position/TimePosition.java create mode 100644 server/pom.xml create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/CanalServer.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/embeded/CanalServerWithEmbeded.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/exception/CanalServerException.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNetty.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/handler/FixedHeaderFrameDecoder.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java create mode 100644 server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandaloneTest.java create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandbyTest.java create mode 100644 server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java create mode 100644 sink/pom.xml create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventDownStreamHandler.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventSink.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventDownStreamHandler.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventSink.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/entry/HeartBeatEntryEventHandler.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupBarrier.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupEventSink.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineBarrier.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineTransactionBarrier.java create mode 100644 sink/src/main/java/com/alibaba/otter/canal/sink/exception/CanalSinkException.java create mode 100644 sink/src/test/java/com/alibaba/otter/canal/sink/GroupEventSinkTest.java create mode 100644 sink/src/test/java/com/alibaba/otter/canal/sink/stub/DummyEventStore.java create mode 100644 store/pom.xml create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalGroupStore.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalStoreScavenge.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/CanalEventStore.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/CanalGroupEventStore.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/CanalStoreConstants.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/CanalStoreException.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/CanalStoreScavenge.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/StoreInfo.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/helper/CanalEventUtils.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/model/BatchMode.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/model/Event.java create mode 100644 store/src/main/java/com/alibaba/otter/canal/store/model/Events.java create mode 100644 store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreBase.java create mode 100644 store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMemBatchTest.java create mode 100644 store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java create mode 100644 store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStorePutAndGetTest.java create mode 100644 store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreRollbackAndAckTest.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2cd141ba --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.svn/ +target/ +test-output/ +*.class +.classpath +.project +.settings/ +tmp +temp +*.log +antx.properties +otter.properties +jtester.properties +.idea/ +*.iml diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..75b52484 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..44640576 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +
+
+ +

最新更新

+
    +
  1. canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。
    2. +
  2. canal消费端项目开源: Otter(分布式数据库同步系统)，地址：https://github.com/alibaba/otter
    3. +
+ +

背景

+

早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&消费的业务，从此开启了一段新纪元。

+

ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.6及以下的版本(阿里内部mysql 5.6.10, mysql 5.5.18和5.1.40/48)

+

+

基于日志增量订阅&消费支持的业务：

+
    +
  1. 数据库镜像
    2. +
  2. 数据库实时备份
    3. +
  3. 多级索引 (卖家和买家各自分库索引)
    4. +
  4. search build
    5. +
  5. 业务cache刷新
    6. +
  6. 价格变化等重要业务消息
    7. +
+

项目介绍

+

名称：canal [kə'næl]

+

译意： 水道/管道/沟渠

+

语言： 纯java开发

+

定位： 基于数据库增量日志解析，提供增量数据订阅&消费，目前主要支持了mysql

+

关键词： mysql binlog parser / real-time / queue&topic

+

+

工作原理

+

mysql主备复制实现

+


从上层来看，复制分成三步： +

    +
  1. master将改变记录到二进制日志(binary log)中（这些记录叫做二进制日志事件，binary log events，可以通过show binlog events进行查看）；
    2. +
  2. slave将master的binary log events拷贝到它的中继日志(relay log)；
    3. +
  3. slave重做中继日志中的事件，将改变反映它自己的数据。
    4. +
+

canal的工作原理：

+

+

原理相对比较简单：

+
    +
  1. canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议
    2. +
  2. mysql master收到dump请求，开始推送binary log给slave(也就是canal)
    3. +
  3. canal解析binary log对象(原始为byte流)
    4. +
+ +

相关文档

+ +See the wiki page for : wiki文档 + +

wiki文档列表

+ + +

问题反馈

+
    +
  1. qq交流群： 161559791
    2. +
  2. 邮件交流： jianghang115@gmail.com
    3. +
  3. 新浪微博： agapple0002
    4. +
  4. 报告issue：issues
    5. +
diff --git a/RELEASE.txt b/RELEASE.txt new file mode 100644 index 00000000..f378f369 --- /dev/null +++ b/RELEASE.txt @@ -0,0 +1,3 @@ +release link : https://alibaba.github.com/canal/release.html + +download link : https://github.com/alibaba/canal/releases diff --git a/client/pom.xml b/client/pom.xml new file mode 100644 index 00000000..efb81197 --- /dev/null +++ b/client/pom.xml @@ -0,0 +1,101 @@ + + 4.0.0 + + com.alibaba.otter + canal + 1.0.19-SNAPSHOT + ../pom.xml + + com.alibaba.otter + canal.client + jar + canal client module for otter ${project.version} + + + com.alibaba.otter + canal.protocol + ${project.version} + + + + + junit + junit + + + + + + dev + + true + + env + !javadoc + + + + + + javadoc + + + env + javadoc + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + attach-javadocs + package + + jar + + + + + true + public + true +
${project.artifactId}-${project.version}
+
${project.artifactId}-${project.version}
+ ${project.artifactId}-${project.version} + + https://github.com/alibaba/canal + + ${project.build.directory}/apidocs/apidocs/${pom.version} + + + + org.apache.maven.plugins + maven-scm-publish-plugin + 1.0-beta-2 + + + attach-javadocs + package + + publish-scm + + + + + ${project.build.directory}/scmpublish + Publishing javadoc for ${project.artifactId}:${project.version} + ${project.build.directory}/apidocs + true + scm:git:git@github.com:alibaba/canal.git + gh-pages + + + + + + + diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java new file mode 100644 index 00000000..b8f84395 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java @@ -0,0 +1,156 @@ +package com.alibaba.otter.canal.client; + +import java.util.concurrent.TimeUnit; + +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + +/** + * canal数据操作客户端 + * + * @author zebin.xuzb @ 2012-6-19 + * @author jianghang + * @version 1.0.0 + */ +public interface CanalConnector { + + /** + * 链接对应的canal server + * + * @throws CanalClientException + */ + public void connect() throws CanalClientException; + + /** + * 释放链接 + * + * @throws CanalClientException + */ + public void disconnect() throws CanalClientException; + + /** + * 检查下链接是否合法 + * + * 
+     * 几种case下链接不合法:
+     * 1. 链接canal server失败，一直没有一个可用的链接，返回false
+     * 2. 当前客户端在进行running抢占的时候，做为备份节点存在，非处于工作节点，返回false
+     * 
+     * 说明：
+     * a. 当前客户端一旦做为备份节点存在，当前所有的对{@linkplain CanalConnector}的操作都会处于阻塞状态，直到转为工作节点
+     * b. 所以业务方最好定时调用checkValid()方法用，比如调用CanalConnector所在线程的interrupt，直接退出CanalConnector，并根据自己的需要退出自己的资源
+     *
+ * + * @throws CanalClientException + */ + public boolean checkValid() throws CanalClientException; + + /** + * 客户端订阅，重复订阅时会更新对应的filter信息 + * + * 
+     * 说明：
+     * a. 如果本次订阅中filter信息为空，则直接使用canal server服务端配置的filter信息
+     * b. 如果本次订阅中filter信息不为空，目前会直接替换canal server服务端配置的filter信息，以本次提交的为准
+     * 
+     * TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
+     *
+ * + * @param clientIdentity + * @throws CanalClientException + */ + void subscribe(String filter) throws CanalClientException; + + /** + * 客户端订阅，不提交客户端filter，以服务端的filter为准 + * + * @param clientIdentity + * @throws CanalClientException + */ + void subscribe() throws CanalClientException; + + /** + * 取消订阅 + * + * @param clientIdentity + * @throws CanalClientException + */ + void unsubscribe() throws CanalClientException; + + /** + * 获取数据，自动进行确认，该方法返回的条件：尝试拿batchSize条记录，有多少取多少，不会阻塞等待 + * + * @param batchSize + * @return + * @throws CanalClientException + */ + Message get(int batchSize) throws CanalClientException; + + /** + * 获取数据，自动进行确认 + * + * 
+     * 该方法返回的条件：
+     *  a. 拿够batchSize条记录或者超过timeout时间
+     *  b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
+     *
+ * + * @param batchSize + * @return + * @throws CanalClientException + */ + Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException; + + /** + * 不指定 position 获取事件，该方法返回的条件: 尝试拿batchSize条记录，有多少取多少，不会阻塞等待
+ * canal 会记住此 client 最新的position。
+ * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。 + * + * @param batchSize + * @throws CanalClientException + */ + Message getWithoutAck(int batchSize) throws CanalClientException; + + /** + * 不指定 position 获取事件. + * + * 
+     * 该方法返回的条件：
+     *  a. 拿够batchSize条记录或者超过timeout时间
+     *  b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
+     *
+ * + * canal 会记住此 client 最新的position。
+ * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。 + * + * @param batchSize + * @param timeout + * @param unit + * @return + * @throws CanalClientException + */ + Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException; + + /** + * 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。 + * + * @param batchId + * @throws CanalClientException + */ + void ack(long batchId) throws CanalClientException; + + /** + * 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId + * + * @throws CanalClientException + */ + void rollback(long batchId) throws CanalClientException; + + /** + * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿 + * + * @throws CanalClientException + */ + void rollback() throws CanalClientException; + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java new file mode 100644 index 00000000..84ef79d9 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java @@ -0,0 +1,70 @@ +package com.alibaba.otter.canal.client; + +import java.net.SocketAddress; +import java.util.List; + +import com.alibaba.otter.canal.client.impl.ClusterCanalConnector; +import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy; +import com.alibaba.otter.canal.client.impl.SimpleCanalConnector; +import com.alibaba.otter.canal.client.impl.SimpleNodeAccessStrategy; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; + +/** + * canal connectors创建工具类 + * + * @author jianghang 2012-10-29 下午11:18:50 + * @version 1.0.0 + */ +public class CanalConnectors { + + /** + * 创建单链接的客户端链接 + * + * @param address + * @param username + * @param password + * @return + */ + public static CanalConnector newSingleConnector(SocketAddress address, String destination, String username, + String password) { + SimpleCanalConnector canalConnector = new SimpleCanalConnector(address, username, password, destination); + canalConnector.setSoTimeout(30 * 1000); + return canalConnector; + } + + /** + * 创建带cluster模式的客户端链接，自动完成failover切换 + * + * @param addresses + * @param username + * @param password + * @return + */ + public static CanalConnector newClusterConnector(List addresses, String destination, + String username, String password) { + ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination, + new SimpleNodeAccessStrategy(addresses)); + canalConnector.setSoTimeout(30 * 1000); + return canalConnector; + } + + /** + * 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描 + * + * @param username + * @param password + * @return + */ + public static CanalConnector newClusterConnector(String zkServers, String destination, String username, + String password) { + ClusterCanalConnector canalConnector = new ClusterCanalConnector( + username, + password, + destination, + new ClusterNodeAccessStrategy( + destination, + ZkClientx.getZkClient(zkServers))); + canalConnector.setSoTimeout(30 * 1000); + return canalConnector; + } +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java b/client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java new file mode 100644 index 00000000..4351e3d2 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java @@ -0,0 +1,14 @@ +package com.alibaba.otter.canal.client; + +import java.net.SocketAddress; + +/** + * 集群节点访问控制接口 + * + * @author jianghang 2012-10-29 下午07:55:41 + * @version 1.0.0 + */ +public interface CanalNodeAccessStrategy { + + SocketAddress nextNode(); +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java new file mode 100644 index 00000000..7309f326 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java @@ -0,0 +1,327 @@ +package com.alibaba.otter.canal.client.impl; + +import java.net.SocketAddress; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.CanalNodeAccessStrategy; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + +/** + * 集群版本connector实现，自带了failover功能
+ * + * @author jianghang 2012-10-29 下午08:04:06 + * @version 1.0.0 + */ +public class ClusterCanalConnector implements CanalConnector { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private String username; + private String password; + private int soTimeout = 10000; + private int retryTimes = 3; + private int retryInterval = 5000; // 重试的时间间隔，默认5秒 + private CanalNodeAccessStrategy accessStrategy; + private SimpleCanalConnector currentConnector; + private String destination; + private String filter; // 记录上一次的filter提交值,便于自动重试时提交 + + public ClusterCanalConnector(String username, String password, String destination, + CanalNodeAccessStrategy accessStrategy){ + this.username = username; + this.password = password; + this.destination = destination; + this.accessStrategy = accessStrategy; + } + + public void connect() throws CanalClientException { + while (currentConnector == null) { + SocketAddress nextAddress = this.accessStrategy.nextNode(); + int times = 0; + while (true) { + try { + currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination); + currentConnector.setSoTimeout(soTimeout); + if (filter != null) { + currentConnector.setFilter(filter); + } + if (accessStrategy instanceof ClusterNodeAccessStrategy) { + currentConnector.setZkClientx(((ClusterNodeAccessStrategy) accessStrategy).getZkClient()); + } + + currentConnector.connect(); + break; + } catch (Exception e) { + logger.warn("failed to connect to:{} after retry {} times", nextAddress, times); + currentConnector.disconnect(); + currentConnector = null; + // retry for #retryTimes for each node when trying to + // connect to it. + times = times + 1; + if (times >= retryTimes) { + throw new CanalClientException(e); + } else { + // fixed issue #55，增加sleep控制，避免重试connect时cpu使用过高 + try { + Thread.sleep(retryInterval); + } catch (InterruptedException e1) { + throw new CanalClientException(e1); + } + } + } + } + } + } + + public boolean checkValid() { + return currentConnector != null && currentConnector.checkValid(); + } + + public void disconnect() throws CanalClientException { + if (currentConnector != null) { + currentConnector.disconnect(); + currentConnector = null; + } + } + + public void subscribe() throws CanalClientException { + subscribe(""); // 传递空字符即可 + } + + public void subscribe(String filter) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + currentConnector.subscribe(filter); + this.filter = filter; + return; + } catch (Throwable t) { + logger.warn("something goes wrong when subscribing from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + + throw new CanalClientException("failed to subscribe after " + times + " times retry."); + } + + public void unsubscribe() throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + currentConnector.unsubscribe(); + return; + } catch (Throwable t) { + logger.warn("something goes wrong when unsubscribing from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + throw new CanalClientException("failed to unsubscribe after " + times + " times retry."); + } + + public Message get(int batchSize) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + Message msg = currentConnector.get(batchSize); + return msg; + } catch (Throwable t) { + logger.warn("something goes wrong when getting data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + throw new CanalClientException("failed to fetch the data after " + times + " times retry"); + } + + public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + Message msg = currentConnector.get(batchSize, timeout, unit); + return msg; + } catch (Throwable t) { + logger.warn("something goes wrong when getting data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + throw new CanalClientException("failed to fetch the data after " + times + " times retry"); + } + + public Message getWithoutAck(int batchSize) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + Message msg = currentConnector.getWithoutAck(batchSize); + return msg; + } catch (Throwable t) { + logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + throw new CanalClientException("failed to fetch the data after " + times + " times retry"); + } + + public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit); + return msg; + } catch (Throwable t) { + logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + throw new CanalClientException("failed to fetch the data after " + times + " times retry"); + } + + public void rollback(long batchId) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + currentConnector.rollback(batchId); + return; + } catch (Throwable t) { + logger.warn("something goes wrong when rollbacking data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + throw new CanalClientException("failed to rollback after " + times + " times retry"); + } + + public void rollback() throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + currentConnector.rollback(); + return; + } catch (Throwable t) { + logger.warn("something goes wrong when rollbacking data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + + throw new CanalClientException("failed to rollback after " + times + " times retry"); + } + + public void ack(long batchId) throws CanalClientException { + int times = 0; + while (times < retryTimes) { + try { + currentConnector.ack(batchId); + return; + } catch (Throwable t) { + logger.warn("something goes wrong when acking data from server:{}\n{}", + currentConnector.getAddress(), + ExceptionUtils.getFullStackTrace(t)); + times++; + restart(); + logger.info("restart the connector for next round retry."); + } + } + + throw new CanalClientException("failed to ack after " + times + " times retry"); + } + + private void restart() throws CanalClientException { + disconnect(); + try { + Thread.sleep(retryInterval); + } catch (InterruptedException e) { + throw new CanalClientException(e); + } + connect(); + } + + // ============================= setter / getter + // ============================ + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public int getSoTimeout() { + return soTimeout; + } + + public void setSoTimeout(int soTimeout) { + this.soTimeout = soTimeout; + } + + public int getRetryTimes() { + return retryTimes; + } + + public void setRetryTimes(int retryTimes) { + this.retryTimes = retryTimes; + } + + public int getRetryInterval() { + return retryInterval; + } + + public void setRetryInterval(int retryInterval) { + this.retryInterval = retryInterval; + } + + public CanalNodeAccessStrategy getAccessStrategy() { + return accessStrategy; + } + + public void setAccessStrategy(CanalNodeAccessStrategy accessStrategy) { + this.accessStrategy = accessStrategy; + } + + public SimpleCanalConnector getCurrentConnector() { + return currentConnector; + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java new file mode 100644 index 00000000..dd68db93 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java @@ -0,0 +1,112 @@ +package com.alibaba.otter.canal.client.impl; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.I0Itec.zkclient.IZkChildListener; +import org.I0Itec.zkclient.IZkDataListener; +import org.apache.commons.lang.StringUtils; + +import com.alibaba.otter.canal.client.CanalNodeAccessStrategy; +import com.alibaba.otter.canal.common.utils.JsonUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; + +/** + * 集群模式的调度策略 + * + * @author jianghang 2012-12-3 下午10:01:04 + * @version 1.0.0 + */ +public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy { + + private IZkChildListener childListener; // 监听所有的服务器列表 + private IZkDataListener dataListener; // 监听当前的工作节点 + private ZkClientx zkClient; + private volatile List currentAddress = new ArrayList(); + private volatile InetSocketAddress runningAddress = null; + + public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){ + this.zkClient = zkClient; + childListener = new IZkChildListener() { + + public void handleChildChange(String parentPath, List currentChilds) throws Exception { + initClusters(currentChilds); + } + + }; + + dataListener = new IZkDataListener() { + + public void handleDataDeleted(String dataPath) throws Exception { + runningAddress = null; + } + + public void handleDataChange(String dataPath, Object data) throws Exception { + initRunning(data); + } + + }; + + String clusterPath = ZookeeperPathUtils.getDestinationClusterRoot(destination); + this.zkClient.subscribeChildChanges(clusterPath, childListener); + initClusters(this.zkClient.getChildren(clusterPath)); + + String runningPath = ZookeeperPathUtils.getDestinationServerRunning(destination); + this.zkClient.subscribeDataChanges(runningPath, dataListener); + initRunning(this.zkClient.readData(runningPath, true)); + } + + public SocketAddress nextNode() { + if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点 + return runningAddress; + } else if (!currentAddress.isEmpty()) { // 如果不存在已经启动的服务，可能服务是一种lazy启动，随机选择一台触发服务器进行启动 + return currentAddress.get(0);// 默认返回第一个节点，之前已经做过shuffle + } else { + throw new CanalClientException("no alive canal server"); + } + } + + private void initClusters(List currentChilds) { + if (currentChilds == null || currentChilds.isEmpty()) { + currentAddress = new ArrayList(); + } else { + List addresses = new ArrayList(); + for (String address : currentChilds) { + String[] strs = StringUtils.split(address, ":"); + if (strs != null && strs.length == 2) { + addresses.add(new InetSocketAddress(strs[0], Integer.valueOf(strs[1]))); + } + } + + Collections.shuffle(addresses); + currentAddress = addresses;// 直接切换引用 + } + } + + private void initRunning(Object data) { + if (data == null) { + return; + } + + ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class); + String[] strs = StringUtils.split(runningData.getAddress(), ':'); + if (strs.length == 2) { + runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1])); + } + } + + public void setZkClient(ZkClientx zkClient) { + this.zkClient = zkClient; + } + + public ZkClientx getZkClient() { + return zkClient; + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java new file mode 100644 index 00000000..7993669d --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java @@ -0,0 +1,475 @@ +package com.alibaba.otter.canal.client.impl; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.SocketChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.client.CanalConnector; +import com.alibaba.otter.canal.client.impl.running.ClientRunningData; +import com.alibaba.otter.canal.client.impl.running.ClientRunningListener; +import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor; +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.utils.BooleanMutex; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.protocol.CanalEntry.Entry; +import com.alibaba.otter.canal.protocol.CanalPacket.Ack; +import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck; +import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth; +import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback; +import com.alibaba.otter.canal.protocol.CanalPacket.Compression; +import com.alibaba.otter.canal.protocol.CanalPacket.Get; +import com.alibaba.otter.canal.protocol.CanalPacket.Handshake; +import com.alibaba.otter.canal.protocol.CanalPacket.Messages; +import com.alibaba.otter.canal.protocol.CanalPacket.Packet; +import com.alibaba.otter.canal.protocol.CanalPacket.PacketType; +import com.alibaba.otter.canal.protocol.CanalPacket.Sub; +import com.alibaba.otter.canal.protocol.CanalPacket.Unsub; +import com.alibaba.otter.canal.protocol.ClientIdentity; +import com.alibaba.otter.canal.protocol.Message; +import com.alibaba.otter.canal.protocol.exception.CanalClientException; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +/** + * 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作 + * + * @author jianghang 2012-10-24 下午05:37:20 + * @version 1.0.0 + */ +public class SimpleCanalConnector implements CanalConnector { + + private static final Logger logger = LoggerFactory.getLogger(SimpleCanalConnector.class); + private SocketAddress address; + private String username; + private String password; + private int soTimeout = 60000; // milliseconds + private String filter; // 记录上一次的filter提交值,便于自动重试时提交 + + private final ByteBuffer readHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN); + private final ByteBuffer writeHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN); + private SocketChannel channel; + private List supportedCompressions = new ArrayList(); + private ClientIdentity clientIdentity; + private ClientRunningMonitor runningMonitor; // 运行控制 + private ZkClientx zkClientx; + private BooleanMutex mutex = new BooleanMutex(false); + private volatile boolean connected = false; // 代表connected是否已正常执行，因为有HA，不代表在工作中 + private boolean rollbackOnConnect = true; // 是否在connect链接成功后，自动执行rollback操作 + private boolean rollbackOnDisConnect = false; // 是否在connect链接成功后，自动执行rollback操作 + + // 读写数据分别使用不同的锁进行控制，减小锁粒度,读也需要排他锁，并发度容易造成数据包混乱，反序列化失败 + private Object readDataLock = new Object(); + private Object writeDataLock = new Object(); + + public SimpleCanalConnector(SocketAddress address, String username, String password, String destination){ + this(address, username, password, destination, 60000); + } + + public SimpleCanalConnector(SocketAddress address, String username, String password, String destination, + int soTimeout){ + this.address = address; + this.username = username; + this.password = password; + this.soTimeout = soTimeout; + this.clientIdentity = new ClientIdentity(destination, (short) 1001); + } + + public void connect() throws CanalClientException { + if (connected) { + return; + } + + if (runningMonitor != null) { + if (!runningMonitor.isStart()) { + runningMonitor.start(); + } + } else { + waitClientRunning(); + doConnect(); + if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次 + subscribe(filter); + } + if (rollbackOnConnect) { + rollback(); + } + } + + connected = true; + } + + public void disconnect() throws CanalClientException { + if (rollbackOnDisConnect && channel.isConnected()) { + rollback(); + } + + connected = false; + if (runningMonitor != null) { + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } else { + doDisconnnect(); + } + } + + private InetSocketAddress doConnect() throws CanalClientException { + try { + channel = SocketChannel.open(); + channel.socket().setSoTimeout(soTimeout); + channel.connect(address); + Packet p = Packet.parseFrom(readNextPacket(channel)); + if (p.getVersion() != 1) { + throw new CanalClientException("unsupported version at this client."); + } + + if (p.getType() != PacketType.HANDSHAKE) { + throw new CanalClientException("expect handshake but found other type."); + } + // + Handshake handshake = Handshake.parseFrom(p.getBody()); + supportedCompressions.addAll(handshake.getSupportedCompressionsList()); + // + ClientAuth ca = ClientAuth.newBuilder() + .setUsername(username != null ? username : "") + .setNetReadTimeout(soTimeout) + .setNetWriteTimeout(soTimeout) + .build(); + writeWithHeader(channel, + Packet.newBuilder() + .setType(PacketType.CLIENTAUTHENTICATION) + .setBody(ca.toByteString()) + .build() + .toByteArray()); + // + Packet ack = Packet.parseFrom(readNextPacket(channel)); + if (ack.getType() != PacketType.ACK) { + throw new CanalClientException("unexpected packet type when ack is expected"); + } + + Ack ackBody = Ack.parseFrom(ack.getBody()); + if (ackBody.getErrorCode() > 0) { + throw new CanalClientException("something goes wrong when doing authentication: " + + ackBody.getErrorMessage()); + } + + connected = true; + return new InetSocketAddress(channel.socket().getLocalAddress(), channel.socket().getLocalPort()); + } catch (IOException e) { + throw new CanalClientException(e); + } + } + + private void doDisconnnect() throws CanalClientException { + if (channel != null) { + try { + channel.close(); + } catch (IOException e) { + logger.warn("exception on closing channel:{} \n {}", channel, e); + } + channel = null; + } + } + + public void subscribe() throws CanalClientException { + subscribe(""); // 传递空字符即可 + } + + public void subscribe(String filter) throws CanalClientException { + waitClientRunning(); + try { + writeWithHeader(channel, + Packet.newBuilder() + .setType(PacketType.SUBSCRIPTION) + .setBody(Sub.newBuilder() + .setDestination(clientIdentity.getDestination()) + .setClientId(String.valueOf(clientIdentity.getClientId())) + .setFilter(filter != null ? filter : "") + .build() + .toByteString()) + .build() + .toByteArray()); + // + Packet p = Packet.parseFrom(readNextPacket(channel)); + Ack ack = Ack.parseFrom(p.getBody()); + if (ack.getErrorCode() > 0) { + throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage()); + } + + clientIdentity.setFilter(filter); + } catch (IOException e) { + throw new CanalClientException(e); + } + } + + public void unsubscribe() throws CanalClientException { + waitClientRunning(); + try { + writeWithHeader(channel, + Packet.newBuilder() + .setType(PacketType.UNSUBSCRIPTION) + .setBody(Unsub.newBuilder() + .setDestination(clientIdentity.getDestination()) + .setClientId(String.valueOf(clientIdentity.getClientId())) + .build() + .toByteString()) + .build() + .toByteArray()); + // + Packet p = Packet.parseFrom(readNextPacket(channel)); + Ack ack = Ack.parseFrom(p.getBody()); + if (ack.getErrorCode() > 0) { + throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage()); + } + } catch (IOException e) { + throw new CanalClientException(e); + } + } + + public Message get(int batchSize) throws CanalClientException { + return get(batchSize, null, null); + } + + public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + Message message = getWithoutAck(batchSize, timeout, unit); + ack(message.getId()); + return message; + } + + public Message getWithoutAck(int batchSize) throws CanalClientException { + return getWithoutAck(batchSize, null, null); + } + + public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException { + waitClientRunning(); + try { + int size = (batchSize <= 0) ? 1000 : batchSize; + long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制 + if (unit == null) { + unit = TimeUnit.MILLISECONDS; + } + + writeWithHeader(channel, + Packet.newBuilder() + .setType(PacketType.GET) + .setBody(Get.newBuilder() + .setAutoAck(false) + .setDestination(clientIdentity.getDestination()) + .setClientId(String.valueOf(clientIdentity.getClientId())) + .setFetchSize(size) + .setTimeout(time) + .setUnit(unit.ordinal()) + .build() + .toByteString()) + .build() + .toByteArray()); + + return receiveMessages(); + } catch (IOException e) { + throw new CanalClientException(e); + } + } + + private Message receiveMessages() throws InvalidProtocolBufferException, IOException { + Packet p = Packet.parseFrom(readNextPacket(channel)); + switch (p.getType()) { + case MESSAGES: { + if (!p.getCompression().equals(Compression.NONE)) { + throw new CanalClientException("compression is not supported in this connector"); + } + + Messages messages = Messages.parseFrom(p.getBody()); + Message result = new Message(messages.getBatchId()); + for (ByteString byteString : messages.getMessagesList()) { + result.addEntry(Entry.parseFrom(byteString)); + } + return result; + } + case ACK: { + Ack ack = Ack.parseFrom(p.getBody()); + throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage()); + } + default: { + throw new CanalClientException("unexpected packet type: " + p.getType()); + } + } + } + + public void ack(long batchId) throws CanalClientException { + waitClientRunning(); + ClientAck ca = ClientAck.newBuilder() + .setDestination(clientIdentity.getDestination()) + .setClientId(String.valueOf(clientIdentity.getClientId())) + .setBatchId(batchId) + .build(); + try { + writeWithHeader(channel, Packet.newBuilder() + .setType(PacketType.CLIENTACK) + .setBody(ca.toByteString()) + .build() + .toByteArray()); + } catch (IOException e) { + throw new CanalClientException(e); + } + } + + public void rollback(long batchId) throws CanalClientException { + waitClientRunning(); + ClientRollback ca = ClientRollback.newBuilder() + .setDestination(clientIdentity.getDestination()) + .setClientId(String.valueOf(clientIdentity.getClientId())) + .setBatchId(batchId) + .build(); + try { + writeWithHeader(channel, Packet.newBuilder() + .setType(PacketType.CLIENTROLLBACK) + .setBody(ca.toByteString()) + .build() + .toByteArray()); + } catch (IOException e) { + throw new CanalClientException(e); + } + } + + public void rollback() throws CanalClientException { + waitClientRunning(); + rollback(0);// 0代笔未设置 + } + + // ==================== helper method ==================== + + private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException { + synchronized (writeDataLock) { + writeHeader.clear(); + writeHeader.putInt(body.length); + writeHeader.flip(); + channel.write(writeHeader); + channel.write(ByteBuffer.wrap(body)); + } + } + + private byte[] readNextPacket(SocketChannel channel) throws IOException { + synchronized (readDataLock) { + readHeader.clear(); + read(channel, readHeader); + int bodyLen = readHeader.getInt(0); + ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen).order(ByteOrder.BIG_ENDIAN); + read(channel, bodyBuf); + return bodyBuf.array(); + } + } + + private void read(SocketChannel channel, ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + int r = channel.read(buffer); + if (r == -1) { + throw new IOException("end of stream when reading header"); + } + } + } + + private synchronized void initClientRunningMonitor(ClientIdentity clientIdentity) { + if (zkClientx != null && clientIdentity != null && runningMonitor == null) { + ClientRunningData clientData = new ClientRunningData(); + clientData.setClientId(clientIdentity.getClientId()); + clientData.setAddress(AddressUtils.getHostIp()); + + runningMonitor = new ClientRunningMonitor(); + runningMonitor.setDestination(clientIdentity.getDestination()); + runningMonitor.setZkClient(zkClientx); + runningMonitor.setClientData(clientData); + runningMonitor.setListener(new ClientRunningListener() { + + public InetSocketAddress processActiveEnter() { + InetSocketAddress address = doConnect(); + mutex.set(true); + if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次 + subscribe(filter); + } + + if (rollbackOnConnect) { + rollback(); + } + + return address; + } + + public void processActiveExit() { + mutex.set(false); + doDisconnnect(); + } + + }); + } + } + + + private void waitClientRunning() { + try { + if (zkClientx != null) { + if (!connected) {// 未调用connect + throw new CanalClientException("should connect first"); + } + + mutex.get();// 阻塞等待 + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CanalClientException(e); + } + } + + public boolean checkValid() { + if (zkClientx != null) { + return mutex.state(); + } else { + return true;// 默认都放过 + } + } + + public SocketAddress getAddress() { + return address; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public int getSoTimeout() { + return soTimeout; + } + + public void setSoTimeout(int soTimeout) { + this.soTimeout = soTimeout; + } + + public void setZkClientx(ZkClientx zkClientx) { + this.zkClientx = zkClientx; + initClientRunningMonitor(this.clientIdentity); + } + + public void setRollbackOnConnect(boolean rollbackOnConnect) { + this.rollbackOnConnect = rollbackOnConnect; + } + + public void setRollbackOnDisConnect(boolean rollbackOnDisConnect) { + this.rollbackOnDisConnect = rollbackOnDisConnect; + } + + public void setFilter(String filter) { + this.filter = filter; + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java new file mode 100644 index 00000000..bb7d9044 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java @@ -0,0 +1,35 @@ +package com.alibaba.otter.canal.client.impl; + +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.otter.canal.client.CanalNodeAccessStrategy; + +/** + * 简单版本的node访问实现 + * + * @author jianghang 2012-10-29 下午08:00:23 + * @version 1.0.0 + */ +public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy { + + private List nodes = new ArrayList(); + private int index = 0; + + public SimpleNodeAccessStrategy(List nodes){ + if (nodes == null || nodes.size() < 1) { + throw new IllegalArgumentException("at least 1 node required."); + } + this.nodes.addAll(nodes); + } + + public SocketAddress nextNode() { + try { + return nodes.get(index); + } finally { + index = (index + 1) % nodes.size(); + } + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningData.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningData.java new file mode 100644 index 00000000..6b41e3e5 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningData.java @@ -0,0 +1,39 @@ +package com.alibaba.otter.canal.client.impl.running; + +/** + * client running状态信息 + * + * @author jianghang 2012-11-22 下午03:41:50 + * @version 1.0.0 + */ +public class ClientRunningData { + + private short clientId; + private String address; + private boolean active = true; + + public short getClientId() { + return clientId; + } + + public void setClientId(short clientId) { + this.clientId = clientId; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public boolean isActive() { + return active; + } + + public void setActive(boolean active) { + this.active = active; + } + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningListener.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningListener.java new file mode 100644 index 00000000..f5ca9853 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningListener.java @@ -0,0 +1,23 @@ +package com.alibaba.otter.canal.client.impl.running; + +import java.net.InetSocketAddress; + +/** + * 触发一下mainstem发生切换 + * + * @author jianghang 2012-9-11 下午02:26:03 + * @version 1.0.0 + */ +public interface ClientRunningListener { + + /** + * 触发现在轮到自己做为active，需要载入上一个active的上下文数据 + */ + public InetSocketAddress processActiveEnter(); + + /** + * 触发一下当前active模式失败 + */ + public void processActiveExit(); + +} diff --git a/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java new file mode 100644 index 00000000..762b9949 --- /dev/null +++ b/client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java @@ -0,0 +1,228 @@ +package com.alibaba.otter.canal.client.impl.running; + +import java.net.InetSocketAddress; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.I0Itec.zkclient.IZkDataListener; +import org.I0Itec.zkclient.exception.ZkException; +import org.I0Itec.zkclient.exception.ZkInterruptedException; +import org.I0Itec.zkclient.exception.ZkNoNodeException; +import org.I0Itec.zkclient.exception.ZkNodeExistsException; +import org.apache.zookeeper.CreateMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.utils.BooleanMutex; +import com.alibaba.otter.canal.common.utils.JsonUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; + +/** + * clinet running控制 + * + * @author jianghang 2012-11-22 下午03:43:01 + * @version 1.0.0 + */ +public class ClientRunningMonitor extends AbstractCanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class); + private ZkClientx zkClient; + private String destination; + private ClientRunningData clientData; + private IZkDataListener dataListener; + private BooleanMutex mutex = new BooleanMutex(false); + private volatile boolean release = false; + private volatile ClientRunningData activeData; + private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); + private ClientRunningListener listener; + private int delayTime = 5; + + public ClientRunningMonitor(){ + dataListener = new IZkDataListener() { + + public void handleDataChange(String dataPath, Object data) throws Exception { + MDC.put("destination", destination); + ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class); + if (!isMine(runningData.getAddress())) { + mutex.set(false); + } + + if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active + release = true; + releaseRunning();// 彻底释放mainstem + } + + activeData = (ClientRunningData) runningData; + } + + public void handleDataDeleted(String dataPath) throws Exception { + MDC.put("destination", destination); + mutex.set(false); + if (!release && activeData != null && isMine(activeData.getAddress())) { + // 如果上一次active的状态就是本机，则即时触发一下active抢占 + initRunning(); + } else { + // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作 + delayExector.schedule(new Runnable() { + + public void run() { + initRunning(); + } + }, delayTime, TimeUnit.SECONDS); + } + } + + }; + + } + + public void start() { + super.start(); + + String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + zkClient.subscribeDataChanges(path, dataListener); + initRunning(); + } + + public void stop() { + super.stop(); + + String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + zkClient.unsubscribeDataChanges(path, dataListener); + releaseRunning(); // 尝试一下release + } + + public void initRunning() { + if (!isStart()) { + return; + } + + String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + // 序列化 + byte[] bytes = JsonUtils.marshalToByte(clientData); + try { + mutex.set(false); + zkClient.create(path, bytes, CreateMode.EPHEMERAL); + processActiveEnter();// 触发一下事件 + activeData = clientData; + mutex.set(true); + } catch (ZkNodeExistsException e) { + bytes = zkClient.readData(path, true); + if (bytes == null) {// 如果不存在节点，立即尝试一次 + initRunning(); + } else { + activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class); + } + } catch (ZkNoNodeException e) { + zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()), + true); // 尝试创建父节点 + initRunning(); + } + } + + /** + * 阻塞等待自己成为active，如果自己成为active，立马返回 + * + * @throws InterruptedException + */ + public void waitForActive() throws InterruptedException { + initRunning(); + mutex.get(); + } + + /** + * 检查当前的状态 + */ + public boolean check() { + String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + try { + byte[] bytes = zkClient.readData(path); + ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class); + activeData = eventData;// 更新下为最新值 + // 检查下nid是否为自己 + boolean result = isMine(activeData.getAddress()); + if (!result) { + logger.warn("canal is running in [{}] , but not in [{}]", + activeData.getAddress(), + clientData.getAddress()); + } + return result; + } catch (ZkNoNodeException e) { + logger.warn("canal is not run any in node"); + return false; + } catch (ZkInterruptedException e) { + logger.warn("canal check is interrupt"); + Thread.interrupted();// 清除interrupt标记 + return check(); + } catch (ZkException e) { + logger.warn("canal check is failed"); + return false; + } + } + + public boolean releaseRunning() { + if (check()) { + String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId()); + zkClient.delete(path); + mutex.set(false); + processActiveExit(); + return true; + } + + return false; + } + + // ====================== helper method ====================== + + private boolean isMine(String address) { + return address.equals(clientData.getAddress()); + } + + private void processActiveEnter() { + if (listener != null) { + // 触发回调，建立与server的socket链接 + InetSocketAddress connectAddress = listener.processActiveEnter(); + String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort(); + this.clientData.setAddress(address); + + String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, + this.clientData.getClientId()); + // 序列化 + byte[] bytes = JsonUtils.marshalToByte(clientData); + zkClient.writeData(path, bytes); + } + } + + private void processActiveExit() { + if (listener != null) { + listener.processActiveExit(); + } + } + + public void setListener(ClientRunningListener listener) { + this.listener = listener; + } + + // ===================== setter / getter ======================= + + public void setDestination(String destination) { + this.destination = destination; + } + + public void setClientData(ClientRunningData clientData) { + this.clientData = clientData; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } + + public void setZkClient(ZkClientx zkClient) { + this.zkClient = zkClient; + } + +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/AbstractZkTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/AbstractZkTest.java new file mode 100644 index 00000000..18cdbc6e --- /dev/null +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/AbstractZkTest.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.client.running; + +import org.junit.Assert; + +public class AbstractZkTest { + + protected String destination = "ljhtest1"; + protected String cluster1 = "127.0.0.1:2188"; + protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188"; + + public void sleep(long time) { + try { + Thread.sleep(time); + } catch (InterruptedException e) { + Assert.fail(e.getMessage()); + } + } +} diff --git a/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java b/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java new file mode 100644 index 00000000..cbdfd0ea --- /dev/null +++ b/client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java @@ -0,0 +1,136 @@ +package com.alibaba.otter.canal.client.running; + +import java.net.InetSocketAddress; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.lang.math.RandomUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.client.impl.running.ClientRunningData; +import com.alibaba.otter.canal.client.impl.running.ClientRunningListener; +import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor; +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; + +public class ClientRunningTest extends AbstractZkTest { + + private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2); + private short clientId = 1001; + + @Before + public void setUp() { + String path = ZookeeperPathUtils.getDestinationPath(destination); + zkclientx.deleteRecursive(path); + + zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true); + } + + @After + public void tearDown() { + String path = ZookeeperPathUtils.getDestinationPath(destination); + zkclientx.deleteRecursive(path); + } + + @Test + public void testOneServer() { + final CountDownLatch countLatch = new CountDownLatch(2); + ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088); + runningMonitor.start(); + sleep(2000L); + runningMonitor.stop(); + sleep(2000L); + + if (countLatch.getCount() != 0) { + Assert.fail(); + } + } + + @Test + public void testMultiServer() { + final CountDownLatch countLatch = new CountDownLatch(30); + final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088); + final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089); + final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090); + final ExecutorService executor = Executors.newFixedThreadPool(3); + executor.submit(new Runnable() { + + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor1.isStart()) { + runningMonitor1.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + runningMonitor1.stop(); + sleep(2000L + RandomUtils.nextInt(500)); + } + } + + }); + + executor.submit(new Runnable() { + + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor2.isStart()) { + runningMonitor2.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + runningMonitor2.stop(); + sleep(2000L + RandomUtils.nextInt(500)); + } + } + + }); + + executor.submit(new Runnable() { + + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor3.isStart()) { + runningMonitor3.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + runningMonitor3.stop(); + sleep(2000L + RandomUtils.nextInt(500)); + } + } + + }); + + sleep(30000L); + } + + private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId, + final int port) { + ClientRunningData clientData = new ClientRunningData(); + clientData.setClientId(clientId); + clientData.setAddress(AddressUtils.getHostIp()); + + ClientRunningMonitor runningMonitor = new ClientRunningMonitor(); + runningMonitor.setDestination(destination); + runningMonitor.setZkClient(zkclientx); + runningMonitor.setClientData(clientData); + runningMonitor.setListener(new ClientRunningListener() { + + public InetSocketAddress processActiveEnter() { + System.out.println(String.format("clientId:%s port:%s has start", clientId, port)); + countLatch.countDown(); + return new InetSocketAddress(AddressUtils.getHostIp(), port); + } + + public void processActiveExit() { + countLatch.countDown(); + System.out.println(String.format("clientId:%s port:%s has stop", clientId, port)); + } + + }); + runningMonitor.setDelayTime(1); + return runningMonitor; + } +} diff --git a/client/src/test/java/logback.xml b/client/src/test/java/logback.xml new file mode 100644 index 00000000..df46c256 --- /dev/null +++ b/client/src/test/java/logback.xml @@ -0,0 +1,14 @@ + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/common/pom.xml b/common/pom.xml new file mode 100644 index 00000000..0a7c3dc7 --- /dev/null +++ b/common/pom.xml @@ -0,0 +1,69 @@ + + 4.0.0 + + com.alibaba.otter + canal + 1.0.19-SNAPSHOT + ../pom.xml + + com.alibaba.otter + canal.common + jar + canal common module for otter ${project.version} + http://b2b-doc.alibaba-inc.com/display/opentech/Otter + + + + org.apache.zookeeper + zookeeper + + + com.github.sgroschupf + zkclient + + + + commons-io + commons-io + + + commons-lang + commons-lang + 2.6 + + + org.springframework + spring + + + com.alibaba + fastjson + + + com.google.guava + guava + + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + junit + junit + + + diff --git a/common/src/main/java/com/alibaba/otter/canal/common/AbstractCanalLifeCycle.java b/common/src/main/java/com/alibaba/otter/canal/common/AbstractCanalLifeCycle.java new file mode 100644 index 00000000..9ceece15 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/AbstractCanalLifeCycle.java @@ -0,0 +1,33 @@ +package com.alibaba.otter.canal.common; + +/** + * 基本实现 + * + * @author jianghang 2012-7-12 上午10:11:07 + * @version 1.0.0 + */ +public abstract class AbstractCanalLifeCycle implements CanalLifeCycle { + + protected volatile boolean running = false; // 是否处于运行中 + + public boolean isStart() { + return running; + } + + public void start() { + if (running) { + throw new CanalException(this.getClass().getName() + " has startup , don't repeat start"); + } + + running = true; + } + + public void stop() { + if (!running) { + throw new CanalException(this.getClass().getName() + " isn't start , please check"); + } + + running = false; + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/CanalException.java b/common/src/main/java/com/alibaba/otter/canal/common/CanalException.java new file mode 100644 index 00000000..71deca09 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/CanalException.java @@ -0,0 +1,37 @@ +package com.alibaba.otter.canal.common; + +import org.apache.commons.lang.exception.NestableRuntimeException; + +/** + * @author jianghang 2012-7-12 上午10:10:31 + * @version 1.0.0 + */ +public class CanalException extends NestableRuntimeException { + + private static final long serialVersionUID = -654893533794556357L; + + public CanalException(String errorCode){ + super(errorCode); + } + + public CanalException(String errorCode, Throwable cause){ + super(errorCode, cause); + } + + public CanalException(String errorCode, String errorDesc){ + super(errorCode + ":" + errorDesc); + } + + public CanalException(String errorCode, String errorDesc, Throwable cause){ + super(errorCode + ":" + errorDesc, cause); + } + + public CanalException(Throwable cause){ + super(cause); + } + + public Throwable fillInStackTrace() { + return this; + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java b/common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java new file mode 100644 index 00000000..6bd15f14 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java @@ -0,0 +1,14 @@ +package com.alibaba.otter.canal.common; + +/** + * @author jianghang 2012-7-12 上午09:39:33 + * @version 1.0.0 + */ +public interface CanalLifeCycle { + + public void start(); + + public void stop(); + + public boolean isStart(); +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java b/common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java new file mode 100644 index 00000000..4c9d1099 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java @@ -0,0 +1,21 @@ +package com.alibaba.otter.canal.common.alarm; + +import com.alibaba.otter.canal.common.CanalLifeCycle; + +/** + * canal报警处理机制 + * + * @author jianghang 2012-8-22 下午10:08:56 + * @version 1.0.0 + */ +public interface CanalAlarmHandler extends CanalLifeCycle { + + /** + * 发送对应destination的报警 + * + * @param destination + * @param title + * @param msg + */ + public void sendAlarm(String destination, String msg); +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/alarm/LogAlarmHandler.java b/common/src/main/java/com/alibaba/otter/canal/common/alarm/LogAlarmHandler.java new file mode 100644 index 00000000..1df22f08 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/alarm/LogAlarmHandler.java @@ -0,0 +1,22 @@ +package com.alibaba.otter.canal.common.alarm; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; + +/** + * 基于log的alarm机制实现 + * + * @author jianghang 2012-8-22 下午10:12:35 + * @version 1.0.0 + */ +public class LogAlarmHandler extends AbstractCanalLifeCycle implements CanalAlarmHandler { + + private static final Logger logger = LoggerFactory.getLogger(LogAlarmHandler.class); + + public void sendAlarm(String destination, String msg) { + logger.error("destination:{}[{}]", new Object[] { destination, msg }); + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java new file mode 100644 index 00000000..d3ee1137 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java @@ -0,0 +1,95 @@ +package com.alibaba.otter.canal.common.utils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.ServerSocket; +import java.util.Enumeration; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AddressUtils { + + private static final Logger logger = LoggerFactory.getLogger(AddressUtils.class); + private static final String LOCALHOST_IP = "127.0.0.1"; + private static final String EMPTY_IP = "0.0.0.0"; + private static final Pattern IP_PATTERN = Pattern.compile("[0-9]{1,3}(\\.[0-9]{1,3}){3,}"); + + public static boolean isAvailablePort(int port) { + ServerSocket ss = null; + try { + ss = new ServerSocket(port); + ss.bind(null); + return true; + } catch (IOException e) { + return false; + } finally { + if (ss != null) { + try { + ss.close(); + } catch (IOException e) { + } + } + } + } + + private static boolean isValidHostAddress(InetAddress address) { + if (address == null || address.isLoopbackAddress()) return false; + String name = address.getHostAddress(); + return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches()); + } + + public static String getHostIp() { + InetAddress address = getHostAddress(); + return address == null ? null : address.getHostAddress(); + } + + public static String getHostName() { + InetAddress address = getHostAddress(); + return address == null ? null : address.getHostName(); + } + + public static InetAddress getHostAddress() { + InetAddress localAddress = null; + try { + localAddress = InetAddress.getLocalHost(); + if (isValidHostAddress(localAddress)) { + return localAddress; + } + } catch (Throwable e) { + logger.warn("Failed to retriving local host ip address, try scan network card ip address. cause: " + + e.getMessage()); + } + try { + Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + if (interfaces != null) { + while (interfaces.hasMoreElements()) { + try { + NetworkInterface network = interfaces.nextElement(); + Enumeration addresses = network.getInetAddresses(); + if (addresses != null) { + while (addresses.hasMoreElements()) { + try { + InetAddress address = addresses.nextElement(); + if (isValidHostAddress(address)) { + return address; + } + } catch (Throwable e) { + logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage()); + } + } + } + } catch (Throwable e) { + logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage()); + } + } + } + } catch (Throwable e) { + logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage()); + } + logger.error("Could not get local host ip address, will use 127.0.0.1 instead."); + return localAddress; + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java new file mode 100644 index 00000000..d729ad01 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java @@ -0,0 +1,157 @@ +package com.alibaba.otter.canal.common.utils; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.AbstractQueuedSynchronizer; + +/** + * 实现一个互斥实现，基于Cocurrent中的{@linkplain AbstractQueuedSynchronizer}实现了自己的sync
+ * 应用场景：系统初始化/授权控制，没权限时阻塞等待。有权限时所有线程都可以快速通过 + * + * 
+ * false : 代表需要被阻塞挂起，等待mutex变为true被唤醒
+ * true : 唤醒被阻塞在false状态下的thread
+ * 
+ * BooleanMutex mutex = new BooleanMutex(true);
+ * try {
+ *     mutex.get(); //当前状态为true, 不会被阻塞
+ * } catch (InterruptedException e) {
+ *     // do something
+ * }
+ * 
+ * mutex.set(false);
+ * try {
+ *     mutex.get(); //当前状态为false, 会被阻塞直到另一个线程调用mutex.set(true);
+ * } catch (InterruptedException e) {
+ *     // do something
+ * }
+ *
+ * + * @author jianghang 2011-9-23 上午09:58:03 + * @version 1.0.0 + */ +public class BooleanMutex { + + private Sync sync; + + public BooleanMutex(){ + sync = new Sync(); + set(false); + } + + public BooleanMutex(Boolean mutex){ + sync = new Sync(); + set(mutex); + } + + /** + * 阻塞等待Boolean为true + * + * @throws InterruptedException + */ + public void get() throws InterruptedException { + sync.innerGet(); + } + + /** + * 阻塞等待Boolean为true,允许设置超时时间 + * + * @param timeout + * @param unit + * @throws InterruptedException + * @throws TimeoutException + */ + public void get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { + sync.innerGet(unit.toNanos(timeout)); + } + + /** + * 重新设置对应的Boolean mutex + * + * @param mutex + */ + public void set(Boolean mutex) { + if (mutex) { + sync.innerSetTrue(); + } else { + sync.innerSetFalse(); + } + } + + public boolean state() { + return sync.innerState(); + } + + /** + * Synchronization control for BooleanMutex. Uses AQS sync state to + * represent run status + */ + private final class Sync extends AbstractQueuedSynchronizer { + + private static final long serialVersionUID = 2559471934544126329L; + /** State value representing that TRUE */ + private static final int TRUE = 1; + /** State value representing that FALSE */ + private static final int FALSE = 2; + + private boolean isTrue(int state) { + return (state & TRUE) != 0; + } + + /** + * 实现AQS的接口，获取共享锁的判断 + */ + protected int tryAcquireShared(int state) { + // 如果为true，直接允许获取锁对象 + // 如果为false，进入阻塞队列，等待被唤醒 + return isTrue(getState()) ? 1 : -1; + } + + /** + * 实现AQS的接口，释放共享锁的判断 + */ + protected boolean tryReleaseShared(int ignore) { + // 始终返回true，代表可以release + return true; + } + + boolean innerState() { + return isTrue(getState()); + } + + void innerGet() throws InterruptedException { + acquireSharedInterruptibly(0); + } + + void innerGet(long nanosTimeout) throws InterruptedException, TimeoutException { + if (!tryAcquireSharedNanos(0, nanosTimeout)) throw new TimeoutException(); + } + + void innerSetTrue() { + for (;;) { + int s = getState(); + if (s == TRUE) { + return; // 直接退出 + } + if (compareAndSetState(s, TRUE)) {// cas更新状态，避免并发更新true操作 + releaseShared(0);// 释放一下锁对象，唤醒一下阻塞的Thread + return; + } + } + } + + void innerSetFalse() { + for (;;) { + int s = getState(); + if (s == FALSE) { + return; // 直接退出 + } + if (compareAndSetState(s, FALSE)) {// cas更新状态，避免并发更新false操作 + return; + } + } + } + + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/CanalToStringStyle.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/CanalToStringStyle.java new file mode 100644 index 00000000..0bc021af --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/CanalToStringStyle.java @@ -0,0 +1,80 @@ +package com.alibaba.otter.canal.common.utils; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import org.apache.commons.lang.builder.ToStringStyle; + +/** + * Otter项目内部使用的ToStringStyle + * + * 
+ * 默认Style输出格式：
+ * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
+ *
+ * + * @author jianghang 2010-6-18 上午11:35:27 + */ +public class CanalToStringStyle extends ToStringStyle { + + private static final long serialVersionUID = -6568177374288222145L; + + private static final String DEFAULT_TIME = "yyyy-MM-dd HH:mm:ss"; + private static final String DEFAULT_DAY = "yyyy-MM-dd"; + + /** + * 
+     * 输出格式：
+     * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
+     *
+ */ + public static final ToStringStyle TIME_STYLE = new OtterDateStyle(DEFAULT_TIME); + + /** + * 
+     * 输出格式：
+     * Person[name=John Doe,age=33,smoker=false ,day=2010-04-01]
+     *
+ */ + public static final ToStringStyle DAY_STYLE = new OtterDateStyle(DEFAULT_DAY); + + /** + * 
+     * 输出格式：
+     * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
+     *
+ */ + public static final ToStringStyle DEFAULT_STYLE = CanalToStringStyle.TIME_STYLE; + + // =========================== 自定义style ============================= + + /** + * 支持日期格式化的ToStringStyle + * + * @author li.jinl + */ + private static class OtterDateStyle extends ToStringStyle { + + private static final long serialVersionUID = 5208917932254652886L; + + // 日期format格式 + private String pattern; + + public OtterDateStyle(String pattern){ + super(); + this.setUseShortClassName(true); + this.setUseIdentityHashCode(false); + // 设置日期format格式 + this.pattern = pattern; + } + + protected void appendDetail(StringBuffer buffer, String fieldName, Object value) { + // 增加自定义的date对象处理 + if (value instanceof Date) { + value = new SimpleDateFormat(pattern).format(value); + } + // 后续可以增加其他自定义对象处理 + buffer.append(value); + } + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java new file mode 100644 index 00000000..cef42308 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java @@ -0,0 +1,73 @@ +package com.alibaba.otter.canal.common.utils; + +import java.util.Arrays; +import java.util.List; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; +import com.alibaba.fastjson.serializer.JSONSerializer; +import com.alibaba.fastjson.serializer.PropertyFilter; +import com.alibaba.fastjson.serializer.SerializeWriter; +import com.alibaba.fastjson.serializer.SerializerFeature; + +/** + * 字节处理相关工具类 + * + * @author jianghang + */ +public class JsonUtils { + + public static T unmarshalFromByte(byte[] bytes, Class targetClass) { + return (T) JSON.parseObject(bytes, targetClass);// 默认为UTF-8 + } + + public static T unmarshalFromByte(byte[] bytes, TypeReference type) { + return (T) JSON.parseObject(bytes, type.getType()); + } + + public static byte[] marshalToByte(Object obj) { + return JSON.toJSONBytes(obj); // 默认为UTF-8 + } + + public static byte[] marshalToByte(Object obj, SerializerFeature... features) { + return JSON.toJSONBytes(obj, features); // 默认为UTF-8 + } + + public static T unmarshalFromString(String json, Class targetClass) { + return (T) JSON.parseObject(json, targetClass);// 默认为UTF-8 + } + + public static T unmarshalFromString(String json, TypeReference type) { + return (T) JSON.parseObject(json, type);// 默认为UTF-8 + } + + public static String marshalToString(Object obj) { + return JSON.toJSONString(obj); // 默认为UTF-8 + } + + public static String marshalToString(Object obj, SerializerFeature... features) { + return JSON.toJSONString(obj, features); // 默认为UTF-8 + } + + /** + * 可以允许指定一些过滤字段进行生成json对象 + */ + public static String marshalToString(Object obj, String... fliterFields) { + final List propertyFliters = Arrays.asList(fliterFields); + SerializeWriter out = new SerializeWriter(); + try { + JSONSerializer serializer = new JSONSerializer(out); + serializer.getPropertyFilters().add(new PropertyFilter() { + + public boolean apply(Object source, String name, Object value) { + return !propertyFliters.contains(name); + } + + }); + serializer.write(obj); + return out.toString(); + } finally { + out.close(); + } + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java new file mode 100644 index 00000000..866664a1 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java @@ -0,0 +1,56 @@ +package com.alibaba.otter.canal.common.utils; + +import java.lang.Thread.UncaughtExceptionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author zebin.xuzb 2012-9-20 下午3:47:47 + * @version 1.0.0 + */ +public class NamedThreadFactory implements ThreadFactory { + + private static final Logger logger = LoggerFactory.getLogger(NamedThreadFactory.class); + final private static String DEFAULT_NAME = "canal-worker"; + final private String name; + final private boolean daemon; + final private ThreadGroup group; + final private AtomicInteger threadNumber = new AtomicInteger(0); + final static UncaughtExceptionHandler uncaughtExceptionHandler = new UncaughtExceptionHandler() { + + public void uncaughtException(Thread t, + Throwable e) { + logger.error("from " + t.getName(), e); + } + }; + + public NamedThreadFactory(){ + this(DEFAULT_NAME, true); + } + + public NamedThreadFactory(String name){ + this(name, true); + } + + public NamedThreadFactory(String name, boolean daemon){ + this.name = name; + this.daemon = daemon; + SecurityManager s = System.getSecurityManager(); + group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); + } + + public Thread newThread(Runnable r) { + Thread t = new Thread(group, r, name + "-" + threadNumber.getAndIncrement(), 0); + t.setDaemon(daemon); + if (t.getPriority() != Thread.NORM_PRIORITY) { + t.setPriority(Thread.NORM_PRIORITY); + } + + t.setUncaughtExceptionHandler(uncaughtExceptionHandler); + return t; + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java new file mode 100644 index 00000000..9522247c --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java @@ -0,0 +1,79 @@ +package com.alibaba.otter.canal.common.utils; + +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Scanner; + +import org.apache.commons.lang.StringUtils; + +/** + * @author zebin.xuzb 2012-11-15 下午3:53:09 + * @since 1.0.0 + */ +public final class UriUtils { + + private final static String SPLIT = "&"; + private final static String EQUAL = "="; + private final static String DEFAULT_ENCODING = "ISO_8859_1"; + + private UriUtils(){ + } + + public static Map parseQuery(final String uriString) { + URI uri = null; + try { + uri = new URI(uriString); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + return parseQuery(uri); + } + + public static Map parseQuery(final String uriString, final String encoding) { + URI uri = null; + try { + uri = new URI(uriString); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + return parseQuery(uri, encoding); + } + + public static Map parseQuery(final URI uri) { + return parseQuery(uri, DEFAULT_ENCODING); + } + + public static Map parseQuery(final URI uri, final String encoding) { + if (uri == null || StringUtils.isBlank(uri.getQuery())) { + return Collections.EMPTY_MAP; + } + String query = uri.getRawQuery(); + HashMap params = new HashMap(); + Scanner scan = new Scanner(query); + scan.useDelimiter(SPLIT); + while (scan.hasNext()) { + String token = scan.next().trim(); + String[] pair = token.split(EQUAL); + String key = decode(pair[0], encoding); + String value = null; + if (pair.length == 2) { + value = decode(pair[1], encoding); + } + params.put(key, value); + } + return params; + } + + private static String decode(final String content, final String encoding) { + try { + return URLDecoder.decode(content, encoding != null ? encoding : DEFAULT_ENCODING); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException(e); + } + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ByteSerializer.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ByteSerializer.java new file mode 100644 index 00000000..caff9c10 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ByteSerializer.java @@ -0,0 +1,32 @@ +package com.alibaba.otter.canal.common.zookeeper; + +import java.io.UnsupportedEncodingException; + +import org.I0Itec.zkclient.exception.ZkMarshallingError; +import org.I0Itec.zkclient.serialize.ZkSerializer; + +/** + * 基于string的序列化方式 + * + * @author jianghang 2012-7-11 下午02:57:09 + * @version 1.0.0 + */ +public class ByteSerializer implements ZkSerializer { + + public Object deserialize(final byte[] bytes) throws ZkMarshallingError { + return bytes; + } + + public byte[] serialize(final Object data) throws ZkMarshallingError { + try { + if (data instanceof byte[]) { + return (byte[]) data; + } else { + return ((String) data).getBytes("utf-8"); + } + } catch (final UnsupportedEncodingException e) { + throw new ZkMarshallingError(e); + } + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/StringSerializer.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/StringSerializer.java new file mode 100644 index 00000000..8309719e --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/StringSerializer.java @@ -0,0 +1,32 @@ +package com.alibaba.otter.canal.common.zookeeper; + +import java.io.UnsupportedEncodingException; + +import org.I0Itec.zkclient.exception.ZkMarshallingError; +import org.I0Itec.zkclient.serialize.ZkSerializer; + +/** + * 基于string的序列化方式 + * + * @author jianghang 2012-7-11 下午02:57:09 + * @version 1.0.0 + */ +public class StringSerializer implements ZkSerializer { + + public Object deserialize(final byte[] bytes) throws ZkMarshallingError { + try { + return new String(bytes, "utf-8"); + } catch (final UnsupportedEncodingException e) { + throw new ZkMarshallingError(e); + } + } + + public byte[] serialize(final Object data) throws ZkMarshallingError { + try { + return ((String) data).getBytes("utf-8"); + } catch (final UnsupportedEncodingException e) { + throw new ZkMarshallingError(e); + } + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java new file mode 100644 index 00000000..cba75cc3 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java @@ -0,0 +1,146 @@ +package com.alibaba.otter.canal.common.zookeeper; + +import java.util.Map; + +import org.I0Itec.zkclient.IZkConnection; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkException; +import org.I0Itec.zkclient.exception.ZkInterruptedException; +import org.I0Itec.zkclient.exception.ZkNoNodeException; +import org.I0Itec.zkclient.exception.ZkNodeExistsException; +import org.I0Itec.zkclient.serialize.ZkSerializer; +import org.apache.zookeeper.CreateMode; + +import com.google.common.base.Function; +import com.google.common.collect.MapMaker; + +/** + * 使用自定义的ZooKeeperx for zk connection + * + * @author jianghang 2012-7-10 下午02:31:15 + * @version 1.0.0 + */ +public class ZkClientx extends ZkClient { + + // 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection + private static Map clients = new MapMaker().makeComputingMap(new Function() { + + public ZkClientx apply(String servers) { + return new ZkClientx(servers); + } + }); + + public static ZkClientx getZkClient(String servers) { + return clients.get(servers); + } + + public ZkClientx(String serverstring){ + this(serverstring, Integer.MAX_VALUE); + } + + public ZkClientx(String zkServers, int connectionTimeout){ + this(new ZooKeeperx(zkServers), connectionTimeout); + } + + public ZkClientx(String zkServers, int sessionTimeout, int connectionTimeout){ + this(new ZooKeeperx(zkServers, sessionTimeout), connectionTimeout); + } + + public ZkClientx(String zkServers, int sessionTimeout, int connectionTimeout, ZkSerializer zkSerializer){ + this(new ZooKeeperx(zkServers, sessionTimeout), connectionTimeout, zkSerializer); + } + + private ZkClientx(IZkConnection connection, int connectionTimeout){ + this(connection, connectionTimeout, new ByteSerializer()); + } + + private ZkClientx(IZkConnection zkConnection, int connectionTimeout, ZkSerializer zkSerializer){ + super(zkConnection, connectionTimeout, zkSerializer); + } + + /** + * Create a persistent Sequential node. + * + * @param path + * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown + * in case the path already exists + * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted + * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread + * @throws ZkException if any ZooKeeper exception occurred + * @throws RuntimeException if any other exception occurs + */ + public String createPersistentSequential(String path, boolean createParents) throws ZkInterruptedException, + IllegalArgumentException, ZkException, + RuntimeException { + try { + return create(path, null, CreateMode.PERSISTENT_SEQUENTIAL); + } catch (ZkNoNodeException e) { + if (!createParents) { + throw e; + } + String parentDir = path.substring(0, path.lastIndexOf('/')); + createPersistent(parentDir, createParents); + return createPersistentSequential(path, createParents); + } + } + + /** + * Create a persistent Sequential node. + * + * @param path + * @param data + * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown + * in case the path already exists + * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted + * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread + * @throws ZkException if any ZooKeeper exception occurred + * @throws RuntimeException if any other exception occurs + */ + public String createPersistentSequential(String path, Object data, boolean createParents) + throws ZkInterruptedException, + IllegalArgumentException, + ZkException, + RuntimeException { + try { + return create(path, data, CreateMode.PERSISTENT_SEQUENTIAL); + } catch (ZkNoNodeException e) { + if (!createParents) { + throw e; + } + String parentDir = path.substring(0, path.lastIndexOf('/')); + createPersistent(parentDir, createParents); + return createPersistentSequential(path, data, createParents); + } + } + + /** + * Create a persistent Sequential node. + * + * @param path + * @param data + * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown + * in case the path already exists + * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted + * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread + * @throws ZkException if any ZooKeeper exception occurred + * @throws RuntimeException if any other exception occurs + */ + public void createPersistent(String path, Object data, boolean createParents) throws ZkInterruptedException, + IllegalArgumentException, ZkException, + RuntimeException { + try { + create(path, data, CreateMode.PERSISTENT); + } catch (ZkNodeExistsException e) { + if (!createParents) { + throw e; + } + } catch (ZkNoNodeException e) { + if (!createParents) { + throw e; + } + String parentDir = path.substring(0, path.lastIndexOf('/')); + createPersistent(parentDir, createParents); + createPersistent(path, data, createParents); + } + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java new file mode 100644 index 00000000..7eec37b3 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java @@ -0,0 +1,181 @@ +package com.alibaba.otter.canal.common.zookeeper; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.I0Itec.zkclient.IZkConnection; +import org.I0Itec.zkclient.exception.ZkException; +import org.apache.commons.lang.StringUtils; +import org.apache.zookeeper.ClientCnxn; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper.States; +import org.apache.zookeeper.client.ConnectStringParser; +import org.apache.zookeeper.client.HostProvider; +import org.apache.zookeeper.client.StaticHostProvider; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.ReflectionUtils; + +/** + * 封装了ZooKeeper，使其支持节点的优先顺序，比如美国机房的节点会优先加载美国对应的zk集群列表，都失败后才会选择加载杭州的zk集群列表 * + * + * @author jianghang 2012-7-10 下午02:31:42 + * @version 1.0.0 + */ +public class ZooKeeperx implements IZkConnection { + + private static final String SERVER_COMMA = ";"; + private static final Logger logger = LoggerFactory.getLogger(ZooKeeperx.class); + private static final Field clientCnxnField = ReflectionUtils.findField(ZooKeeper.class, "cnxn"); + private static final Field hostProviderField = ReflectionUtils.findField(ClientCnxn.class, "hostProvider"); + private static final Field serverAddressesField = ReflectionUtils.findField(StaticHostProvider.class, + "serverAddresses"); + private static final int DEFAULT_SESSION_TIMEOUT = 90000; + + private ZooKeeper _zk = null; + private Lock _zookeeperLock = new ReentrantLock(); + + private final List _servers; + private final int _sessionTimeOut; + + public ZooKeeperx(String zkServers){ + this(zkServers, DEFAULT_SESSION_TIMEOUT); + } + + public ZooKeeperx(String zkServers, int sessionTimeOut){ + _servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA)); + _sessionTimeOut = sessionTimeOut; + } + + @Override + public void connect(Watcher watcher) { + _zookeeperLock.lock(); + try { + if (_zk != null) { + throw new IllegalStateException("zk client has already been started"); + } + + try { + logger.debug("Creating new ZookKeeper instance to connect to " + _servers + "."); + _zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher); + configMutliCluster(_zk); + } catch (IOException e) { + throw new ZkException("Unable to connect to " + _servers, e); + } + } finally { + _zookeeperLock.unlock(); + } + } + + public void close() throws InterruptedException { + _zookeeperLock.lock(); + try { + if (_zk != null) { + logger.debug("Closing ZooKeeper connected to " + _servers); + _zk.close(); + _zk = null; + } + } finally { + _zookeeperLock.unlock(); + } + } + + public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException { + return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode); + } + + public void delete(String path) throws InterruptedException, KeeperException { + _zk.delete(path, -1); + } + + public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException { + return _zk.exists(path, watch) != null; + } + + public List getChildren(final String path, final boolean watch) throws KeeperException, + InterruptedException { + return _zk.getChildren(path, watch); + } + + public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException { + return _zk.getData(path, watch, stat); + } + + public void writeData(String path, byte[] data) throws KeeperException, InterruptedException { + writeData(path, data, -1); + } + + public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException { + _zk.setData(path, data, version); + } + + public States getZookeeperState() { + return _zk != null ? _zk.getState() : null; + } + + public ZooKeeper getZookeeper() { + return _zk; + } + + public long getCreateTime(String path) throws KeeperException, InterruptedException { + Stat stat = _zk.exists(path, false); + if (stat != null) { + return stat.getCtime(); + } + return -1; + } + + public String getServers() { + return StringUtils.join(_servers, SERVER_COMMA); + } + + // =============================== + + public void configMutliCluster(ZooKeeper zk) { + if (_servers.size() == 1) { + return; + } + String cluster1 = _servers.get(0); + try { + if (_servers.size() > 1) { + // 强制的声明accessible + ReflectionUtils.makeAccessible(clientCnxnField); + ReflectionUtils.makeAccessible(hostProviderField); + ReflectionUtils.makeAccessible(serverAddressesField); + + // 添加第二组集群列表 + for (int i = 1; i < _servers.size(); i++) { + String cluster = _servers.get(i); + // 强制获取zk中的地址信息 + ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk); + HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn); + List serverAddrs = (List) ReflectionUtils.getField( + serverAddressesField, + hostProvider); + // 添加第二组集群列表 + serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses()); + } + } + } catch (Exception e) { + try { + if (zk != null) { + zk.close(); + } + } catch (InterruptedException ie) { + // ignore interrupt + } + throw new ZkException("zookeeper_create_error, serveraddrs=" + cluster1, e); + } + + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java new file mode 100644 index 00000000..967e0f29 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java @@ -0,0 +1,181 @@ +package com.alibaba.otter.canal.common.zookeeper; + +import java.text.MessageFormat; + +import org.apache.commons.lang.StringUtils; + +/** + * 存储结构： + * + * 
+ * /otter
+ *    canal
+ *      cluster
+ *      destinations
+ *        dest1
+ *          running (EPHEMERAL) 
+ *          cluster
+ *          client1
+ *            running (EPHEMERAL)
+ *            cluster
+ *            filter
+ *            cursor
+ *            mark
+ *              1
+ *              2
+ *              3
+ *
+ * + * @author zebin.xuzb @ 2012-6-21 + * @version 1.0.0 + */ +public class ZookeeperPathUtils { + + public static final String ZOOKEEPER_SEPARATOR = "/"; + + public static final String OTTER_ROOT_NODE = ZOOKEEPER_SEPARATOR + "otter"; + + public static final String CANAL_ROOT_NODE = OTTER_ROOT_NODE + ZOOKEEPER_SEPARATOR + + "canal"; + + public static final String DESTINATION_ROOT_NODE = CANAL_ROOT_NODE + ZOOKEEPER_SEPARATOR + + "destinations"; + + public static final String FILTER_NODE = "filter"; + + public static final String BATCH_MARK_NODE = "mark"; + + public static final String PARSE_NODE = "parse"; + + public static final String CURSOR_NODE = "cursor"; + + public static final String RUNNING_NODE = "running"; + + public static final String CLUSTER_NODE = "cluster"; + + public static final String DESTINATION_NODE = DESTINATION_ROOT_NODE + + ZOOKEEPER_SEPARATOR + "{0}"; + + public static final String DESTINATION_PARSE_NODE = DESTINATION_NODE + ZOOKEEPER_SEPARATOR + + PARSE_NODE; + + public static final String DESTINATION_CLIENTID_NODE = DESTINATION_NODE + ZOOKEEPER_SEPARATOR + + "{1}"; + + public static final String DESTINATION_CURSOR_NODE = DESTINATION_CLIENTID_NODE + + ZOOKEEPER_SEPARATOR + CURSOR_NODE; + + public static final String DESTINATION_CLIENTID_FILTER_NODE = DESTINATION_CLIENTID_NODE + + ZOOKEEPER_SEPARATOR + FILTER_NODE; + + public static final String DESTINATION_CLIENTID_BATCH_MARK_NODE = DESTINATION_CLIENTID_NODE + + ZOOKEEPER_SEPARATOR + BATCH_MARK_NODE; + + public static final String DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH = DESTINATION_CLIENTID_BATCH_MARK_NODE + + ZOOKEEPER_SEPARATOR + "{2}"; + + /** + * 服务端当前正在提供服务的running节点 + */ + public static final String DESTINATION_RUNNING_NODE = DESTINATION_NODE + ZOOKEEPER_SEPARATOR + + RUNNING_NODE; + + /** + * 客户端当前正在工作的running节点 + */ + public static final String DESTINATION_CLIENTID_RUNNING_NODE = DESTINATION_CLIENTID_NODE + + ZOOKEEPER_SEPARATOR + RUNNING_NODE; + + /** + * 整个canal server的集群列表 + */ + public static final String CANAL_CLUSTER_ROOT_NODE = CANAL_ROOT_NODE + ZOOKEEPER_SEPARATOR + + CLUSTER_NODE; + + public static final String CANAL_CLUSTER_NODE = CANAL_CLUSTER_ROOT_NODE + + ZOOKEEPER_SEPARATOR + "{0}"; + + /** + * 针对某个destination的工作的集群列表 + */ + public static final String DESTINATION_CLUSTER_ROOT = DESTINATION_NODE + ZOOKEEPER_SEPARATOR + + CLUSTER_NODE; + public static final String DESTINATION_CLUSTER_NODE = DESTINATION_CLUSTER_ROOT + + ZOOKEEPER_SEPARATOR + "{1}"; + + public static String getDestinationPath(String destinationName) { + return MessageFormat.format(DESTINATION_NODE, destinationName); + } + + public static String getClientIdNodePath(String destinationName, short clientId) { + return MessageFormat.format(DESTINATION_CLIENTID_NODE, destinationName, String.valueOf(clientId)); + } + + public static String getFilterPath(String destinationName, short clientId) { + return MessageFormat.format(DESTINATION_CLIENTID_FILTER_NODE, destinationName, String.valueOf(clientId)); + } + + public static String getBatchMarkPath(String destinationName, short clientId) { + return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_NODE, destinationName, String.valueOf(clientId)); + } + + public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) { + return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName, + String.valueOf(clientId), getBatchMarkNode(batchId)); + } + + public static String getCursorPath(String destination, short clientId) { + return MessageFormat.format(DESTINATION_CURSOR_NODE, destination, String.valueOf(clientId)); + } + + public static String getCanalClusterNode(String node) { + return MessageFormat.format(CANAL_CLUSTER_NODE, node); + } + + /** + * 服务端当前正在提供服务的running节点 + */ + public static String getDestinationServerRunning(String destination) { + return MessageFormat.format(DESTINATION_RUNNING_NODE, destination); + } + + /** + * 客户端当前正在工作的running节点 + */ + public static String getDestinationClientRunning(String destination, short clientId) { + return MessageFormat.format(DESTINATION_CLIENTID_RUNNING_NODE, destination, String.valueOf(clientId)); + } + + public static String getDestinationClusterNode(String destination, String node) { + return MessageFormat.format(DESTINATION_CLUSTER_NODE, destination, node); + } + + public static String getDestinationClusterRoot(String destination) { + return MessageFormat.format(DESTINATION_CLUSTER_ROOT, destination); + } + + public static String getParsePath(String destination) { + return MessageFormat.format(DESTINATION_PARSE_NODE, destination); + } + + /** + * 将batchNode转换为Long + */ + public static short getClientId(String clientNode) { + return Short.valueOf(clientNode); + } + + /** + * 将batchNode转换为Long + */ + public static long getBatchMarkId(String batchMarkNode) { + return Long.valueOf(batchMarkNode); + } + + /** + * 将batchId转化为zookeeper中的node名称 + */ + public static String getBatchMarkNode(Long batchId) { + return StringUtils.leftPad(String.valueOf(batchId.intValue()), 10, '0'); + } +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningData.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningData.java new file mode 100644 index 00000000..6ff20427 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningData.java @@ -0,0 +1,59 @@ +package com.alibaba.otter.canal.common.zookeeper.running; + +import java.io.Serializable; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; + +/** + * 服务端running状态信息 + * + * @author jianghang 2012-11-22 下午03:11:30 + * @version 1.0.0 + */ +public class ServerRunningData implements Serializable { + + private static final long serialVersionUID = 92260481691855281L; + + private Long cid; + private String address; + private boolean active = true; + + public ServerRunningData(){ + } + + public ServerRunningData(Long cid, String address){ + this.cid = cid; + this.address = address; + } + + public Long getCid() { + return cid; + } + + public void setCid(Long cid) { + this.cid = cid; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public boolean isActive() { + return active; + } + + public void setActive(boolean active) { + this.active = active; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningListener.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningListener.java new file mode 100644 index 00000000..4fe34ca3 --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningListener.java @@ -0,0 +1,31 @@ +package com.alibaba.otter.canal.common.zookeeper.running; + +/** + * 触发一下mainstem发生切换 + * + * @author jianghang 2012-9-11 下午02:26:03 + * @version 1.0.0 + */ +public interface ServerRunningListener { + + /** + * 启动时回调做点事情 + */ + public void processStart(); + + /** + * 关闭时回调做点事情 + */ + public void processStop(); + + /** + * 触发现在轮到自己做为active，需要载入上一个active的上下文数据 + */ + public void processActiveEnter(); + + /** + * 触发一下当前active模式失败 + */ + public void processActiveExit(); + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java new file mode 100644 index 00000000..e11771bc --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java @@ -0,0 +1,273 @@ +package com.alibaba.otter.canal.common.zookeeper.running; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.I0Itec.zkclient.IZkDataListener; +import org.I0Itec.zkclient.exception.ZkException; +import org.I0Itec.zkclient.exception.ZkInterruptedException; +import org.I0Itec.zkclient.exception.ZkNoNodeException; +import org.I0Itec.zkclient.exception.ZkNodeExistsException; +import org.apache.zookeeper.CreateMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.utils.BooleanMutex; +import com.alibaba.otter.canal.common.utils.JsonUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; + +/** + * 针对server的running节点控制 + * + * @author jianghang 2012-11-22 下午02:59:42 + * @version 1.0.0 + */ +public class ServerRunningMonitor extends AbstractCanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(ServerRunningMonitor.class); + private ZkClientx zkClient; + private String destination; + private IZkDataListener dataListener; + private BooleanMutex mutex = new BooleanMutex(false); + private volatile boolean release = false; + // 当前服务节点状态信息 + private ServerRunningData serverData; + // 当前实际运行的节点状态信息 + private volatile ServerRunningData activeData; + private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1); + private int delayTime = 5; + private ServerRunningListener listener; + + public ServerRunningMonitor(ServerRunningData serverData){ + this(); + this.serverData = serverData; + } + + public ServerRunningMonitor(){ + // 创建父节点 + dataListener = new IZkDataListener() { + + public void handleDataChange(String dataPath, Object data) throws Exception { + MDC.put("destination", destination); + ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class); + if (!isMine(runningData.getAddress())) { + mutex.set(false); + } + + if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active + release = true; + releaseRunning();// 彻底释放mainstem + } + + activeData = (ServerRunningData) runningData; + } + + public void handleDataDeleted(String dataPath) throws Exception { + MDC.put("destination", destination); + mutex.set(false); + if (!release && activeData != null && isMine(activeData.getAddress())) { + // 如果上一次active的状态就是本机，则即时触发一下active抢占 + initRunning(); + } else { + // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作 + delayExector.schedule(new Runnable() { + + public void run() { + initRunning(); + } + }, delayTime, TimeUnit.SECONDS); + } + } + + }; + + } + + public void start() { + super.start(); + processStart(); + if (zkClient != null) { + // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start + String path = ZookeeperPathUtils.getDestinationServerRunning(destination); + zkClient.subscribeDataChanges(path, dataListener); + + initRunning(); + } else { + processActiveEnter();// 没有zk，直接启动 + } + } + + public void release() { + if (zkClient != null) { + releaseRunning(); // 尝试一下release + } else { + processActiveExit(); // 没有zk，直接启动 + } + } + + public void stop() { + super.stop(); + + if (zkClient != null) { + String path = ZookeeperPathUtils.getDestinationServerRunning(destination); + zkClient.unsubscribeDataChanges(path, dataListener); + + releaseRunning(); // 尝试一下release + } else { + processActiveExit(); // 没有zk，直接启动 + } + processStop(); + } + + private void initRunning() { + if (!isStart()) { + return; + } + + String path = ZookeeperPathUtils.getDestinationServerRunning(destination); + // 序列化 + byte[] bytes = JsonUtils.marshalToByte(serverData); + try { + mutex.set(false); + zkClient.create(path, bytes, CreateMode.EPHEMERAL); + activeData = serverData; + processActiveEnter();// 触发一下事件 + mutex.set(true); + } catch (ZkNodeExistsException e) { + bytes = zkClient.readData(path, true); + if (bytes == null) {// 如果不存在节点，立即尝试一次 + initRunning(); + } else { + activeData = JsonUtils.unmarshalFromByte(bytes, ServerRunningData.class); + } + } catch (ZkNoNodeException e) { + zkClient.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true); // 尝试创建父节点 + initRunning(); + } + } + + /** + * 阻塞等待自己成为active，如果自己成为active，立马返回 + * + * @throws InterruptedException + */ + public void waitForActive() throws InterruptedException { + initRunning(); + mutex.get(); + } + + /** + * 检查当前的状态 + */ + public boolean check() { + String path = ZookeeperPathUtils.getDestinationServerRunning(destination); + try { + byte[] bytes = zkClient.readData(path); + ServerRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ServerRunningData.class); + activeData = eventData;// 更新下为最新值 + // 检查下nid是否为自己 + boolean result = isMine(activeData.getAddress()); + if (!result) { + logger.warn("canal is running in node[{}] , but not in node[{}]", + activeData.getCid(), + serverData.getCid()); + } + return result; + } catch (ZkNoNodeException e) { + logger.warn("canal is not run any in node"); + return false; + } catch (ZkInterruptedException e) { + logger.warn("canal check is interrupt"); + Thread.interrupted();// 清除interrupt标记 + return check(); + } catch (ZkException e) { + logger.warn("canal check is failed"); + return false; + } + } + + private boolean releaseRunning() { + if (check()) { + String path = ZookeeperPathUtils.getDestinationServerRunning(destination); + zkClient.delete(path); + mutex.set(false); + processActiveExit(); + return true; + } + + return false; + } + + // ====================== helper method ====================== + + private boolean isMine(String address) { + return address.equals(serverData.getAddress()); + } + + private void processStart() { + if (listener != null) { + try { + listener.processStart(); + } catch (Exception e) { + logger.error("processStart failed", e); + } + } + } + + private void processStop() { + if (listener != null) { + try { + listener.processStop(); + } catch (Exception e) { + logger.error("processStop failed", e); + } + } + } + + private void processActiveEnter() { + if (listener != null) { + try { + listener.processActiveEnter(); + } catch (Exception e) { + logger.error("processActiveEnter failed", e); + } + } + } + + private void processActiveExit() { + if (listener != null) { + try { + listener.processActiveExit(); + } catch (Exception e) { + logger.error("processActiveExit failed", e); + } + } + } + + public void setListener(ServerRunningListener listener) { + this.listener = listener; + } + + // ===================== setter / getter ======================= + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } + + public void setServerData(ServerRunningData serverData) { + this.serverData = serverData; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public void setZkClient(ZkClientx zkClient) { + this.zkClient = zkClient; + } + +} diff --git a/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java new file mode 100644 index 00000000..30569ebb --- /dev/null +++ b/common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java @@ -0,0 +1,36 @@ +package com.alibaba.otter.canal.common.zookeeper.running; + +import java.util.Map; + +/** + * {@linkplain ServerRunningMonitor}管理容器，使用static进行数据全局共享 + * + * @author jianghang 2012-12-3 下午09:32:06 + * @version 1.0.0 + */ +public class ServerRunningMonitors { + + private static ServerRunningData serverData; + private static Map runningMonitors; // + + public static ServerRunningData getServerData() { + return serverData; + } + + public static Map getRunningMonitors() { + return runningMonitors; + } + + public static ServerRunningMonitor getRunningMonitor(String destination) { + return (ServerRunningMonitor) runningMonitors.get(destination); + } + + public static void setServerData(ServerRunningData serverData) { + ServerRunningMonitors.serverData = serverData; + } + + public static void setRunningMonitors(Map runningMonitors) { + ServerRunningMonitors.runningMonitors = runningMonitors; + } + +} diff --git a/common/src/test/java/com/alibaba/otter/canal/common/AbstractZkTest.java b/common/src/test/java/com/alibaba/otter/canal/common/AbstractZkTest.java new file mode 100644 index 00000000..cf5c4681 --- /dev/null +++ b/common/src/test/java/com/alibaba/otter/canal/common/AbstractZkTest.java @@ -0,0 +1,18 @@ +package com.alibaba.otter.canal.common; + +import org.junit.Assert; + +public class AbstractZkTest { + + protected String destination = "ljhtest1"; + protected String cluster1 = "127.0.0.1:2188"; + protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188"; + + public void sleep(long time) { + try { + Thread.sleep(time); + } catch (InterruptedException e) { + Assert.fail(e.getMessage()); + } + } +} diff --git a/common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java b/common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java new file mode 100644 index 00000000..8f1e72e8 --- /dev/null +++ b/common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java @@ -0,0 +1,143 @@ +package com.alibaba.otter.canal.common; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.commons.lang.math.RandomUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor; + +public class ServerRunningTest extends AbstractZkTest { + + private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2); + + @Before + public void setUp() { + String path = ZookeeperPathUtils.getDestinationPath(destination); + zkclientx.deleteRecursive(path); + + zkclientx.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true); + } + + @After + public void tearDown() { + String path = ZookeeperPathUtils.getDestinationPath(destination); + zkclientx.deleteRecursive(path); + } + + @Test + public void testOneServer() { + final CountDownLatch countLatch = new CountDownLatch(2); + ServerRunningMonitor runningMonitor = buildServerRunning(countLatch, 1L, "127.0.0.1", 2088); + runningMonitor.start(); + sleep(2000L); + runningMonitor.stop(); + sleep(2000L); + + if (countLatch.getCount() != 0) { + Assert.fail(); + } + } + + @Test + public void testMultiServer() { + final CountDownLatch countLatch = new CountDownLatch(30); + final ServerRunningMonitor runningMonitor1 = buildServerRunning(countLatch, 1L, "127.0.0.1", 2088); + final ServerRunningMonitor runningMonitor2 = buildServerRunning(countLatch, 2L, "127.0.0.1", 2089); + final ServerRunningMonitor runningMonitor3 = buildServerRunning(countLatch, 3L, "127.0.0.1", 2090); + final ExecutorService executor = Executors.newFixedThreadPool(3); + executor.submit(new Runnable() { + + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor1.isStart()) { + runningMonitor1.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + if (runningMonitor1.check()) { + runningMonitor1.stop(); + } + sleep(2000L + RandomUtils.nextInt(500)); + } + } + + }); + + executor.submit(new Runnable() { + + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor2.isStart()) { + runningMonitor2.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + if (runningMonitor2.check()) { + runningMonitor2.stop(); + } + sleep(2000L + RandomUtils.nextInt(500)); + } + } + + }); + + executor.submit(new Runnable() { + + public void run() { + for (int i = 0; i < 10; i++) { + if (!runningMonitor3.isStart()) { + runningMonitor3.start(); + } + sleep(2000L + RandomUtils.nextInt(500)); + if (runningMonitor3.check()) { + runningMonitor3.stop(); + } + sleep(2000L + RandomUtils.nextInt(500)); + } + } + + }); + + sleep(30000L); + } + + private ServerRunningMonitor buildServerRunning(final CountDownLatch countLatch, final Long cid, final String ip, + final int port) { + ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port); + ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); + runningMonitor.setDestination(destination); + runningMonitor.setListener(new ServerRunningListener() { + + public void processActiveEnter() { + System.out.println(String.format("cid:%s ip:%s:%s has start", cid, ip, port)); + countLatch.countDown(); + } + + public void processActiveExit() { + System.out.println(String.format("cid:%s ip:%s:%s has stop", cid, ip, port)); + countLatch.countDown(); + } + + public void processStart() { + System.out.println(String.format("cid:%s ip:%s:%s processStart", cid, ip, port)); + } + + public void processStop() { + System.out.println(String.format("cid:%s ip:%s:%s processStop", cid, ip, port)); + } + + }); + + runningMonitor.setZkClient(zkclientx); + runningMonitor.setDelayTime(1); + return runningMonitor; + } +} diff --git a/common/src/test/java/logback.xml b/common/src/test/java/logback.xml new file mode 100644 index 00000000..df46c256 --- /dev/null +++ b/common/src/test/java/logback.xml @@ -0,0 +1,14 @@ + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/dbsync/pom.xml b/dbsync/pom.xml new file mode 100644 index 00000000..c8e60279 --- /dev/null +++ b/dbsync/pom.xml @@ -0,0 +1,43 @@ + + 4.0.0 + + com.alibaba.otter + canal + 1.0.19-SNAPSHOT + ../pom.xml + + com.alibaba.otter + canal.parse.dbsync + jar + canal dbsync module for otter ${project.version} + + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + + junit + junit + test + + + mysql + mysql-connector-java + test + + + diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java new file mode 100644 index 00000000..b9f3dade --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java @@ -0,0 +1,367 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.nio.charset.Charset; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * An utility class implements MySQL/Java charsets conversion. you can see com.mysql.jdbc.CharsetMapping. + * + * @author Changyuan.lh + */ +public final class CharsetConversion +{ + static final Log logger = LogFactory.getLog(CharsetConversion.class); + + static final class Entry + { + protected final int charsetId; + protected final String mysqlCharset; + protected final String mysqlCollation; + protected final String javaCharset; + + Entry(final int id, String mysqlCharset, // NL + String mysqlCollation, String javaCharset) + { + this.charsetId = id; + this.mysqlCharset = mysqlCharset; + this.mysqlCollation = mysqlCollation; + this.javaCharset = javaCharset; + } + } + + // Character set data used in lookups. The array will be sparse. + static final Entry[] entries = new Entry[0xff]; + + static Entry getEntry(final int id) + { + if (id >= 0 && id < entries.length) + { + return entries[id]; + } + else + { + throw new IllegalArgumentException("Invalid charset id: " + id); + } + } + + // Loads character set information. + static void putEntry(final int charsetId, String mysqlCharset, + String mysqlCollation, String javaCharset) + { + entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL + mysqlCollation, javaCharset); + } + + // Loads character set information. + @Deprecated + static void putEntry(final int charsetId, String mysqlCharset, + String mysqlCollation) + { + entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL + mysqlCollation, /* Unknown java charset */null); + } + + // Load character set data statically. + static + { + putEntry(1, "big5", "big5_chinese_ci", "Big5"); + putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2"); + putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1"); + putEntry(4, "cp850", "cp850_general_ci", "Cp850"); + putEntry(5, "latin1", "latin1_german1_ci", "ISO8859_1"); + putEntry(6, "hp8", "hp8_english_ci", "ISO8859_1"); + putEntry(7, "koi8r", "koi8r_general_ci", "KOI8_R"); + putEntry(8, "latin1", "latin1_swedish_ci", "ISO8859_1"); + putEntry(9, "latin2", "latin2_general_ci", "ISO8859_2"); + putEntry(10, "swe7", "swe7_swedish_ci", "ISO8859_1"); + putEntry(11, "ascii", "ascii_general_ci", "US-ASCII"); + putEntry(12, "ujis", "ujis_japanese_ci", "EUC_JP"); + putEntry(13, "sjis", "sjis_japanese_ci", "SJIS"); + putEntry(14, "cp1251", "cp1251_bulgarian_ci", "Cp1251"); + putEntry(15, "latin1", "latin1_danish_ci", "ISO8859_1"); + putEntry(16, "hebrew", "hebrew_general_ci", "ISO8859_8"); + putEntry(17, "filename", "filename", "ISO8859_1"); + putEntry(18, "tis620", "tis620_thai_ci", "TIS620"); + putEntry(19, "euckr", "euckr_korean_ci", "EUC_KR"); + putEntry(20, "latin7", "latin7_estonian_cs", "ISO8859_7"); + putEntry(21, "latin2", "latin2_hungarian_ci", "ISO8859_2"); + putEntry(22, "koi8u", "koi8u_general_ci", "KOI8_U"); + putEntry(23, "cp1251", "cp1251_ukrainian_ci", "Cp1251"); + putEntry(24, "gb2312", "gb2312_chinese_ci", "EUC_CN"); + putEntry(25, "greek", "greek_general_ci", "ISO8859_7"); + putEntry(26, "cp1250", "cp1250_general_ci", "Cp1250"); + putEntry(27, "latin2", "latin2_croatian_ci", "ISO8859_2"); + putEntry(28, "gbk", "gbk_chinese_ci", "GBK"); + putEntry(29, "cp1257", "cp1257_lithuanian_ci", "Cp1257"); + putEntry(30, "latin5", "latin5_turkish_ci", "ISO8859_5"); + putEntry(31, "latin1", "latin1_german2_ci", "ISO8859_1"); + putEntry(32, "armscii8", "armscii8_general_ci", "ISO8859_1"); + putEntry(33, "utf8", "utf8_general_ci", "UTF-8"); + putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250"); + putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig"); + putEntry(36, "cp866", "cp866_general_ci", "Cp866"); + putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp895"); + putEntry(38, "macce", "macce_general_ci", "MacCentralEurope"); + putEntry(39, "macroman", "macroman_general_ci", "MacRoman"); + putEntry(40, "cp852", "cp852_general_ci", "Cp852"); + putEntry(41, "latin7", "latin7_general_ci", "ISO8859_7"); + putEntry(42, "latin7", "latin7_general_cs", "ISO8859_7"); + putEntry(43, "macce", "macce_bin", "MacCentralEurope"); + putEntry(44, "cp1250", "cp1250_croatian_ci", "Cp1250"); + putEntry(45, "utf8mb4", "utf8mb4_general_ci", "MacCentralEurope"); + putEntry(46, "utf8mb4", "utf8mb4_bin", "MacCentralEurope"); + putEntry(47, "latin1", "latin1_bin", "ISO8859_1"); + putEntry(48, "latin1", "latin1_general_ci", "ISO8859_1"); + putEntry(49, "latin1", "latin1_general_cs", "ISO8859_1"); + putEntry(50, "cp1251", "cp1251_bin", "Cp1251"); + putEntry(51, "cp1251", "cp1251_general_ci", "Cp1251"); + putEntry(52, "cp1251", "cp1251_general_cs", "Cp1251"); + putEntry(53, "macroman", "macroman_bin", "MacRoman"); + putEntry(54, "utf16", "utf16_general_ci", "UTF-16"); + putEntry(55, "utf16", "utf16_bin", "UTF-16"); + putEntry(57, "cp1256", "cp1256_general_ci", "Cp1256"); + putEntry(58, "cp1257", "cp1257_bin", "Cp1257"); + putEntry(59, "cp1257", "cp1257_general_ci", "Cp1257"); + putEntry(60, "utf32", "utf32_general_ci", "UTF-32"); + putEntry(61, "utf32", "utf32_bin", "UTF-32"); + putEntry(63, "binary", "binary", "US-ASCII"); + putEntry(64, "armscii8", "armscii8_bin", "ISO8859_2"); + putEntry(65, "ascii", "ascii_bin", "US-ASCII"); + putEntry(66, "cp1250", "cp1250_bin", "Cp1250"); + putEntry(67, "cp1256", "cp1256_bin", "Cp1256"); + putEntry(68, "cp866", "cp866_bin", "Cp866"); + putEntry(69, "dec8", "dec8_bin", "US-ASCII"); + putEntry(70, "greek", "greek_bin", "ISO8859_7"); + putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8"); + putEntry(72, "hp8", "hp8_bin" , "US-ASCII"); + putEntry(73, "keybcs2", "keybcs2_bin", "Cp895"); + putEntry(74, "koi8r", "koi8r_bin", "KOI8_R"); + putEntry(75, "koi8u", "koi8u_bin", "KOI8_U"); + putEntry(77, "latin2", "latin2_bin", "ISO8859_2"); + putEntry(78, "latin5", "latin5_bin", "ISO8859_5"); + putEntry(79, "latin7", "latin7_bin", "ISO8859_7"); + putEntry(80, "cp850", "cp850_bin", "Cp850"); + putEntry(81, "cp852", "cp852_bin", "Cp852"); + putEntry(82, "swe7", "swe7_bin", "ISO8859_1"); + putEntry(83, "utf8", "utf8_bin", "UTF-8"); + putEntry(84, "big5", "big5_bin", "Big5"); + putEntry(85, "euckr", "euckr_bin", "EUC_KR"); + putEntry(86, "gb2312", "gb2312_bin", "EUC_CN"); + putEntry(87, "gbk", "gbk_bin", "GBK"); + putEntry(88, "sjis", "sjis_bin", "SJIS"); + putEntry(89, "tis620", "tis620_bin", "TIS620"); + putEntry(90, "ucs2", "ucs2_bin", "UnicodeBig"); + putEntry(91, "ujis", "ujis_bin", "EUC_JP"); + putEntry(92, "geostd8", "geostd8_general_ci", "US-ASCII"); + putEntry(93, "geostd8", "geostd8_bin", "US-ASCII"); + putEntry(94, "latin1", "latin1_spanish_ci", "ISO8859_1"); + putEntry(95, "cp932", "cp932_japanese_ci", "Shift_JIS"); + putEntry(96, "cp932", "cp932_bin", "Shift_JIS"); + putEntry(97, "eucjpms", "eucjpms_japanese_ci", "EUC_JP"); + putEntry(98, "eucjpms", "eucjpms_bin", "EUC_JP"); + putEntry(99, "cp1250", "cp1250_polish_ci", "Cp1250"); + + putEntry(101, "utf16", "utf16_unicode_ci", "UTF-16"); + putEntry(102, "utf16", "utf16_icelandic_ci", "UTF-16"); + putEntry(103, "utf16", "utf16_latvian_ci", "UTF-16"); + putEntry(104, "utf16", "utf16_romanian_ci", "UTF-16"); + putEntry(105, "utf16", "utf16_slovenian_ci", "UTF-16"); + putEntry(106, "utf16", "utf16_polish_ci", "UTF-16"); + putEntry(107, "utf16", "utf16_estonian_ci", "UTF-16"); + putEntry(108, "utf16", "utf16_spanish_ci", "UTF-16"); + putEntry(109, "utf16", "utf16_swedish_ci", "UTF-16"); + putEntry(110, "utf16", "utf16_turkish_ci", "UTF-16"); + putEntry(111, "utf16", "utf16_czech_ci", "UTF-16"); + putEntry(112, "utf16", "utf16_danish_ci", "UTF-16"); + putEntry(113, "utf16", "utf16_lithuanian_ci", "UTF-16"); + putEntry(114, "utf16", "utf16_slovak_ci", "UTF-16"); + putEntry(115, "utf16", "utf16_spanish2_ci", "UTF-16"); + putEntry(116, "utf16", "utf16_roman_ci", "UTF-16"); + putEntry(117, "utf16", "utf16_persian_ci", "UTF-16"); + putEntry(118, "utf16", "utf16_esperanto_ci", "UTF-16"); + putEntry(119, "utf16", "utf16_hungarian_ci", "UTF-16"); + putEntry(120, "utf16", "utf16_sinhala_ci", "UTF-16"); + + putEntry(128, "ucs2", "ucs2_unicode_ci", "UnicodeBig"); + putEntry(129, "ucs2", "ucs2_icelandic_ci", "UnicodeBig"); + putEntry(130, "ucs2", "ucs2_latvian_ci", "UnicodeBig"); + putEntry(131, "ucs2", "ucs2_romanian_ci", "UnicodeBig"); + putEntry(132, "ucs2", "ucs2_slovenian_ci", "UnicodeBig"); + putEntry(133, "ucs2", "ucs2_polish_ci", "UnicodeBig"); + putEntry(134, "ucs2", "ucs2_estonian_ci", "UnicodeBig"); + putEntry(135, "ucs2", "ucs2_spanish_ci", "UnicodeBig"); + putEntry(136, "ucs2", "ucs2_swedish_ci", "UnicodeBig"); + putEntry(137, "ucs2", "ucs2_turkish_ci", "UnicodeBig"); + putEntry(138, "ucs2", "ucs2_czech_ci", "UnicodeBig"); + putEntry(139, "ucs2", "ucs2_danish_ci", "UnicodeBig"); + putEntry(140, "ucs2", "ucs2_lithuanian_ci", "UnicodeBig"); + putEntry(141, "ucs2", "ucs2_slovak_ci", "UnicodeBig"); + putEntry(142, "ucs2", "ucs2_spanish2_ci", "UnicodeBig"); + putEntry(143, "ucs2", "ucs2_roman_ci", "UnicodeBig"); + putEntry(144, "ucs2", "ucs2_persian_ci", "UnicodeBig"); + putEntry(145, "ucs2", "ucs2_esperanto_ci", "UnicodeBig"); + putEntry(146, "ucs2", "ucs2_hungarian_ci", "UnicodeBig"); + putEntry(147, "ucs2", "ucs2_sinhala_ci", "UnicodeBig"); + + putEntry(160, "utf32", "utf32_unicode_ci", "UTF-32"); + putEntry(161, "utf32", "utf32_icelandic_ci", "UTF-32"); + putEntry(162, "utf32", "utf32_latvian_ci", "UTF-32"); + putEntry(163, "utf32", "utf32_romanian_ci", "UTF-32"); + putEntry(164, "utf32", "utf32_slovenian_ci", "UTF-32"); + putEntry(165, "utf32", "utf32_polish_ci", "UTF-32"); + putEntry(166, "utf32", "utf32_estonian_ci", "UTF-32"); + putEntry(167, "utf32", "utf32_spanish_ci", "UTF-32"); + putEntry(168, "utf32", "utf32_swedish_ci", "UTF-32"); + putEntry(169, "utf32", "utf32_turkish_ci", "UTF-32"); + putEntry(170, "utf32", "utf32_czech_ci", "UTF-32"); + putEntry(171, "utf32", "utf32_danish_ci", "UTF-32"); + putEntry(172, "utf32", "utf32_lithuanian_ci", "UTF-32"); + putEntry(173, "utf32", "utf32_slovak_ci", "UTF-32"); + putEntry(174, "utf32", "utf32_spanish2_ci", "UTF-32"); + putEntry(175, "utf32", "utf32_roman_ci", "UTF-32"); + putEntry(176, "utf32", "utf32_persian_ci", "UTF-32"); + putEntry(177, "utf32", "utf32_esperanto_ci", "UTF-32"); + putEntry(178, "utf32", "utf32_hungarian_ci", "UTF-32"); + putEntry(179, "utf32", "utf32_sinhala_ci", "UTF-32"); + + putEntry(192, "utf8", "utf8_unicode_ci", "UTF-8"); + putEntry(193, "utf8", "utf8_icelandic_ci", "UTF-8"); + putEntry(194, "utf8", "utf8_latvian_ci", "UTF-8"); + putEntry(195, "utf8", "utf8_romanian_ci", "UTF-8"); + putEntry(196, "utf8", "utf8_slovenian_ci", "UTF-8"); + putEntry(197, "utf8", "utf8_polish_ci", "UTF-8"); + putEntry(198, "utf8", "utf8_estonian_ci", "UTF-8"); + putEntry(199, "utf8", "utf8_spanish_ci", "UTF-8"); + putEntry(200, "utf8", "utf8_swedish_ci", "UTF-8"); + putEntry(201, "utf8", "utf8_turkish_ci", "UTF-8"); + putEntry(202, "utf8", "utf8_czech_ci", "UTF-8"); + putEntry(203, "utf8", "utf8_danish_ci", "UTF-8"); + putEntry(204, "utf8", "utf8_lithuanian_ci", "UTF-8"); + putEntry(205, "utf8", "utf8_slovak_ci", "UTF-8"); + putEntry(206, "utf8", "utf8_spanish2_ci", "UTF-8"); + putEntry(207, "utf8", "utf8_roman_ci", "UTF-8"); + putEntry(208, "utf8", "utf8_persian_ci", "UTF-8"); + putEntry(209, "utf8", "utf8_esperanto_ci", "UTF-8"); + putEntry(210, "utf8", "utf8_hungarian_ci", "UTF-8"); + putEntry(211, "utf8", "utf8_sinhala_ci", "UTF-8"); + + putEntry(224, "utf8mb4", "utf8mb4_unicode_ci", "UTF-8"); + putEntry(225, "utf8mb4", "utf8mb4_icelandic_ci", "UTF-8"); + putEntry(226, "utf8mb4", "utf8mb4_latvian_ci", "UTF-8"); + putEntry(227, "utf8mb4", "utf8mb4_romanian_ci", "UTF-8"); + putEntry(228, "utf8mb4", "utf8mb4_slovenian_ci", "UTF-8"); + putEntry(229, "utf8mb4", "utf8mb4_polish_ci", "UTF-8"); + putEntry(230, "utf8mb4", "utf8mb4_estonian_ci", "UTF-8"); + putEntry(231, "utf8mb4", "utf8mb4_spanish_ci", "UTF-8"); + putEntry(232, "utf8mb4", "utf8mb4_swedish_ci", "UTF-8"); + putEntry(233, "utf8mb4", "utf8mb4_turkish_ci", "UTF-8"); + putEntry(234, "utf8mb4", "utf8mb4_czech_ci", "UTF-8"); + putEntry(235, "utf8mb4", "utf8mb4_danish_ci", "UTF-8"); + putEntry(236, "utf8mb4", "utf8mb4_lithuanian_ci", "UTF-8"); + putEntry(237, "utf8mb4", "utf8mb4_slovak_ci", "UTF-8"); + putEntry(238, "utf8mb4", "utf8mb4_spanish2_ci", "UTF-8"); + putEntry(239, "utf8mb4", "utf8mb4_roman_ci", "UTF-8"); + putEntry(240, "utf8mb4", "utf8mb4_persian_ci", "UTF-8"); + putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8"); + putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8"); + putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8"); + + putEntry(254, "utf8", "utf8_general_cs", "UTF-8"); + } + + /** + * Return defined charset name for mysql. + */ + public static String getCharset(final int id) + { + Entry entry = getEntry(id); + + if (entry != null) + { + return entry.mysqlCharset; + } + else + { + logger.warn("Unexpect mysql charset: " + id); + return null; + } + } + + /** + * Return defined collaction name for mysql. + */ + public static String getCollation(final int id) + { + Entry entry = getEntry(id); + + if (entry != null) + { + return entry.mysqlCollation; + } + else + { + logger.warn("Unexpect mysql charset: " + id); + return null; + } + } + + /** + * Return converted charset name for java. + */ + public static String getJavaCharset(final int id) + { + Entry entry = getEntry(id); + + if (entry != null) + { + if (entry.javaCharset != null) + { + return entry.javaCharset; + } + else + { + logger.warn("Unknown java charset for: id = " + id + + ", name = " + entry.mysqlCharset + ", coll = " + + entry.mysqlCollation); + return null; + } + } + else + { + logger.warn("Unexpect mysql charset: " + id); + return null; + } + } + + public static void main(String[] args) + { + for (int i = 0; i < entries.length; i++) + { + Entry entry = entries[i]; + + System.out.print(i); + System.out.print(','); + System.out.print(' '); + if (entry != null) + { + System.out.print(entry.mysqlCharset); + System.out.print(','); + System.out.print(' '); + System.out.print(entry.javaCharset); + if (entry.javaCharset != null) + { + System.out.print(','); + System.out.print(' '); + System.out.print(Charset.forName(entry.javaCharset).name()); + } + } + else + { + System.out.print("null"); + } + System.out.println(); + } + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java new file mode 100644 index 00000000..70f763a8 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java @@ -0,0 +1,487 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.SocketTimeoutException; +import java.sql.Connection; +import java.sql.SQLException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * TODO: Document It!! + * + * 
+ * DirectLogFetcher fetcher = new DirectLogFetcher();
+ * fetcher.open(conn, file, 0, 13);
+ * 
+ * while (fetcher.fetch())
+ * {
+ *     LogEvent event;
+ *     do
+ *     {
+ *         event = decoder.decode(fetcher, context);
+ * 
+ *         // process log event.
+ *     }
+ *     while (event != null);
+ * }
+ * // connection closed.
+ *
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class DirectLogFetcher extends LogFetcher +{ + protected static final Log logger = LogFactory.getLog(DirectLogFetcher.class); + + /** Command to dump binlog */ + public static final byte COM_BINLOG_DUMP = 18; + + /** Packet header sizes */ + public static final int NET_HEADER_SIZE = 4; + public static final int SQLSTATE_LENGTH = 5; + + /** Packet offsets */ + public static final int PACKET_LEN_OFFSET = 0; + public static final int PACKET_SEQ_OFFSET = 3; + + /** Maximum packet length */ + public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1); + + /** BINLOG_DUMP options */ + public static final int BINLOG_DUMP_NON_BLOCK = 1; + public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2; + + private Connection conn; + private OutputStream mysqlOutput; + private InputStream mysqlInput; + + public DirectLogFetcher() + { + super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR); + } + + public DirectLogFetcher(final int initialCapacity) + { + super(initialCapacity, DEFAULT_GROWTH_FACTOR); + } + + public DirectLogFetcher(final int initialCapacity, final float growthFactor) + { + super(initialCapacity, growthFactor); + } + + private static final Object unwrapConnection(Object conn, Class connClazz) + throws IOException + { + while (!connClazz.isInstance(conn)) + { + try + { + Class connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy"); + if (connProxy.isInstance(conn)) + { + conn = invokeMethod(conn, connProxy, "getTargetConnection"); + continue; + } + } + catch (ClassNotFoundException e) + { + // org.springframework.jdbc.datasource.ConnectionProxy not found. + } + + try + { + Class connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection"); + if (connProxy.isInstance(conn)) + { + conn = getDeclaredField(conn, connProxy, "_conn"); + continue; + } + } + catch (ClassNotFoundException e) + { + // org.apache.commons.dbcp.DelegatingConnection not found. + } + + try + { + if (conn instanceof java.sql.Wrapper) + { + Class connIface = Class.forName("com.mysql.jdbc.Connection"); + conn = ((java.sql.Wrapper) conn).unwrap(connIface); + continue; + } + } + catch (ClassNotFoundException e) + { + // com.mysql.jdbc.Connection not found. + } + catch (SQLException e) + { + logger.warn("Unwrap " + conn.getClass().getName() + " to " + + connClazz.getName() + " failed: " + e.getMessage(), e); + } + + return null; + } + return conn; + } + + private static final Object invokeMethod(Object obj, Class objClazz, + String name) + { + try + { + Method method = objClazz.getMethod(name, (Class[]) null); + return method.invoke(obj, (Object[]) null); + } + catch (NoSuchMethodException e) + { + throw new IllegalArgumentException("No such method: \'" + name + + "\' @ " + objClazz.getName(), e); + } + catch (IllegalAccessException e) + { + throw new IllegalArgumentException("Cannot invoke method: \'" + + name + "\' @ " + objClazz.getName(), e); + } + catch (InvocationTargetException e) + { + throw new IllegalArgumentException("Invoke method failed: \'" + + name + "\' @ " + objClazz.getName(), + e.getTargetException()); + } + } + + private static final Object getDeclaredField(Object obj, Class objClazz, + String name) + { + try + { + Field field = objClazz.getDeclaredField(name); + field.setAccessible(true); + return field.get(obj); + } + catch (NoSuchFieldException e) + { + throw new IllegalArgumentException("No such field: \'" + name + + "\' @ " + objClazz.getName(), e); + } + catch (IllegalAccessException e) + { + throw new IllegalArgumentException("Cannot get field: \'" + name + + "\' @ " + objClazz.getName(), e); + } + } + + /** + * Connect MySQL master to fetch binlog. + */ + public void open(Connection conn, String fileName, final int serverId) + throws IOException + { + open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false); + } + + /** + * Connect MySQL master to fetch binlog. + */ + public void open(Connection conn, String fileName, final int serverId, + boolean nonBlocking) throws IOException + { + open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking); + } + + /** + * Connect MySQL master to fetch binlog. + */ + public void open(Connection conn, String fileName, final long filePosition, + final int serverId) throws IOException + { + open(conn, fileName, filePosition, serverId, false); + } + + /** + * Connect MySQL master to fetch binlog. + */ + public void open(Connection conn, String fileName, long filePosition, + final int serverId, boolean nonBlocking) throws IOException + { + try + { + this.conn = conn; + Class connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl"); + Object unwrapConn = unwrapConnection(conn, connClazz); + if (unwrapConn == null) + { + throw new IOException("Unable to unwrap " + + conn.getClass().getName() + + " to com.mysql.jdbc.ConnectionImpl"); + } + + // Get underlying IO streams for network communications. + Object connIo = getDeclaredField(unwrapConn, connClazz, "io"); + if (connIo == null) + { + throw new IOException("Get null field:" + + conn.getClass().getName() + "#io"); + } + mysqlOutput = (OutputStream) getDeclaredField(connIo, + connIo.getClass(), "mysqlOutput"); + mysqlInput = (InputStream) getDeclaredField(connIo, + connIo.getClass(), "mysqlInput"); + + if (filePosition == 0) + filePosition = BIN_LOG_HEADER_SIZE; + sendBinlogDump(fileName, filePosition, serverId, nonBlocking); + position = 0; + } + catch (IOException e) + { + close(); /* Do cleanup */ + logger.error("Error on COM_BINLOG_DUMP: file = " + fileName + + ", position = " + filePosition); + throw e; + } + catch (ClassNotFoundException e) + { + close(); /* Do cleanup */ + throw new IOException( + "Unable to load com.mysql.jdbc.ConnectionImpl", e); + } + } + + /** + * Put a byte in the buffer. + * + * @param b the byte to put in the buffer + */ + protected final void putByte(byte b) + { + ensureCapacity(position + 1); + + buffer[position++] = b; + } + + /** + * Put 16-bit integer in the buffer. + * + * @param i16 the integer to put in the buffer + */ + protected final void putInt16(int i16) + { + ensureCapacity(position + 2); + + byte[] buf = buffer; + buf[position++] = (byte) (i16 & 0xff); + buf[position++] = (byte) (i16 >>> 8); + } + + /** + * Put 32-bit integer in the buffer. + * + * @param i32 the integer to put in the buffer + */ + protected final void putInt32(long i32) + { + ensureCapacity(position + 4); + + byte[] buf = buffer; + buf[position++] = (byte) (i32 & 0xff); + buf[position++] = (byte) (i32 >>> 8); + buf[position++] = (byte) (i32 >>> 16); + buf[position++] = (byte) (i32 >>> 24); + } + + /** + * Put a string in the buffer. + * + * @param s the value to put in the buffer + */ + protected final void putString(String s) + { + ensureCapacity(position + (s.length() * 2) + 1); + + System.arraycopy(s.getBytes(), 0, buffer, position, s.length()); + position += s.length(); + buffer[position++] = 0; + } + + protected final void sendBinlogDump(String fileName, + final long filePosition, final int serverId, boolean nonBlocking) + throws IOException + { + position = NET_HEADER_SIZE; + + putByte(COM_BINLOG_DUMP); + putInt32(filePosition); + int binlog_flags = nonBlocking ? BINLOG_DUMP_NON_BLOCK : 0; + binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT; + putInt16(binlog_flags); // binlog_flags + putInt32(serverId); // slave's server-id + putString(fileName); + + final byte[] buf = buffer; + final int len = position - NET_HEADER_SIZE; + buf[0] = (byte) (len & 0xff); + buf[1] = (byte) (len >>> 8); + buf[2] = (byte) (len >>> 16); + + mysqlOutput.write(buffer, 0, position); + mysqlOutput.flush(); + } + + /** + * {@inheritDoc} + * + * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch() + */ + public boolean fetch() throws IOException + { + try + { + // Fetching packet header from input. + if (!fetch0(0, NET_HEADER_SIZE)) + { + logger.warn("Reached end of input stream while fetching header"); + return false; + } + + // Fetching the first packet(may a multi-packet). + int netlen = getUint24(PACKET_LEN_OFFSET); + int netnum = getUint8(PACKET_SEQ_OFFSET); + if (!fetch0(NET_HEADER_SIZE, netlen)) + { + logger.warn("Reached end of input stream: packet #" + netnum + + ", len = " + netlen); + return false; + } + + // Detecting error code. + final int mark = getUint8(NET_HEADER_SIZE); + if (mark != 0) + { + if (mark == 255) // error from master + { + // Indicates an error, for example trying to fetch from wrong + // binlog position. + position = NET_HEADER_SIZE + 1; + final int errno = getInt16(); + String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH); + String errmsg = getFixString(limit - position); + throw new IOException("Received error packet:" + + " errno = " + errno + ", sqlstate = " + sqlstate + + " errmsg = " + errmsg); + } + else if (mark == 254) + { + // Indicates end of stream. It's not clear when this would + // be sent. + logger.warn("Received EOF packet from server, apparent" + + " master disconnected."); + return false; + } + else + { + // Should not happen. + throw new IOException("Unexpected response " + mark + + " while fetching binlog: packet #" + netnum + + ", len = " + netlen); + } + } + + // The first packet is a multi-packet, concatenate the packets. + while (netlen == MAX_PACKET_LENGTH) + { + if (!fetch0(0, NET_HEADER_SIZE)) + { + logger.warn("Reached end of input stream while fetching header"); + return false; + } + + netlen = getUint24(PACKET_LEN_OFFSET); + netnum = getUint8(PACKET_SEQ_OFFSET); + if (!fetch0(limit, netlen)) + { + logger.warn("Reached end of input stream: packet #" + + netnum + ", len = " + netlen); + return false; + } + } + + // Preparing buffer variables to decoding. + origin = NET_HEADER_SIZE + 1; + position = origin; + limit -= origin; + return true; + } + catch (SocketTimeoutException e) + { + close(); /* Do cleanup */ + logger.error("Socket timeout expired, closing connection", e); + throw e; + } + catch (InterruptedIOException e) + { + close(); /* Do cleanup */ + logger.warn("I/O interrupted while reading from client socket", e); + throw e; + } + catch (IOException e) + { + close(); /* Do cleanup */ + logger.error("I/O error while reading from client socket", e); + throw e; + } + } + + private final boolean fetch0(final int off, final int len) + throws IOException + { + ensureCapacity(off + len); + + for (int count, n = 0; n < len; n += count) + { + if (0 > (count = mysqlInput.read(buffer, off + n, len - n))) + { + // Reached end of input stream + return false; + } + } + + if (limit < off + len) + limit = off + len; + return true; + } + + /** + * {@inheritDoc} + * + * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close() + */ + public void close() throws IOException + { + try + { + if (conn != null) + conn.close(); + + conn = null; + mysqlInput = null; + mysqlOutput = null; + } + catch (SQLException e) + { + logger.warn("Unable to close connection", e); + } + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java new file mode 100644 index 00000000..dce0ee6b --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java @@ -0,0 +1,160 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; + +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; + +/** + * TODO: Document It!! + * + * 
+ * FileLogFetcher fetcher = new FileLogFetcher();
+ * fetcher.open(file, 0);
+ * 
+ * while (fetcher.fetch()) {
+ *     LogEvent event;
+ *     do {
+ *         event = decoder.decode(fetcher, context);
+ * 
+ *         // process log event.
+ *     } while (event != null);
+ * }
+ * // file ending reached.
+ *
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class FileLogFetcher extends LogFetcher { + + public static final byte[] BINLOG_MAGIC = { -2, 0x62, 0x69, 0x6e }; + + private FileInputStream fin; + + public FileLogFetcher(){ + super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR); + } + + public FileLogFetcher(final int initialCapacity){ + super(initialCapacity, DEFAULT_GROWTH_FACTOR); + } + + public FileLogFetcher(final int initialCapacity, final float growthFactor){ + super(initialCapacity, growthFactor); + } + + /** + * Open binlog file in local disk to fetch. + */ + public void open(File file) throws FileNotFoundException, IOException { + open(file, 0L); + } + + /** + * Open binlog file in local disk to fetch. + */ + public void open(String filePath) throws FileNotFoundException, IOException { + open(new File(filePath), 0L); + } + + /** + * Open binlog file in local disk to fetch. + */ + public void open(String filePath, final long filePosition) throws FileNotFoundException, IOException { + open(new File(filePath), filePosition); + } + + /** + * Open binlog file in local disk to fetch. + */ + public void open(File file, final long filePosition) throws FileNotFoundException, IOException { + fin = new FileInputStream(file); + + ensureCapacity(BIN_LOG_HEADER_SIZE); + if (BIN_LOG_HEADER_SIZE != fin.read(buffer, 0, BIN_LOG_HEADER_SIZE)) throw new IOException("No binlog file header"); + + if (buffer[0] != BINLOG_MAGIC[0] || buffer[1] != BINLOG_MAGIC[1] || buffer[2] != BINLOG_MAGIC[2] + || buffer[3] != BINLOG_MAGIC[3]) { + throw new IOException("Error binlog file header: " + + Arrays.toString(Arrays.copyOf(buffer, BIN_LOG_HEADER_SIZE))); + } + + limit = 0; + origin = 0; + position = 0; + + if (filePosition > BIN_LOG_HEADER_SIZE) { + final int maxFormatDescriptionEventLen = FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN + + FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET + + LogEvent.ENUM_END_EVENT + LogEvent.BINLOG_CHECKSUM_ALG_DESC_LEN + + LogEvent.CHECKSUM_CRC32_SIGNATURE_LEN; + + ensureCapacity(maxFormatDescriptionEventLen); + limit = fin.read(buffer, 0, maxFormatDescriptionEventLen); + limit = (int) getUint32(LogEvent.EVENT_LEN_OFFSET); + fin.getChannel().position(filePosition); + } + } + + /** + * {@inheritDoc} + * + * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch() + */ + public boolean fetch() throws IOException { + if (limit == 0) { + final int len = fin.read(buffer, 0, buffer.length); + if (len >= 0) { + limit += len; + position = 0; + origin = 0; + + /* More binlog to fetch */ + return true; + } + } else if (origin == 0) { + if (limit > buffer.length / 2) { + ensureCapacity(buffer.length + limit); + } + final int len = fin.read(buffer, limit, buffer.length - limit); + if (len >= 0) { + limit += len; + + /* More binlog to fetch */ + return true; + } + } else if (limit > 0) { + System.arraycopy(buffer, origin, buffer, 0, limit); + final int len = fin.read(buffer, limit, buffer.length - limit); + if (len >= 0) { + limit += len; + position -= origin; + origin = 0; + + /* More binlog to fetch */ + return true; + } + } else { + /* Should not happen. */ + throw new IllegalArgumentException("Unexcepted limit: " + limit); + } + + /* Reach binlog file end */ + return false; + } + + /** + * {@inheritDoc} + * + * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close() + */ + public void close() throws IOException { + if (fin != null) fin.close(); + + fin = null; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java new file mode 100644 index 00000000..d975777f --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java @@ -0,0 +1,1995 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.BitSet; + +/** + * TODO: Document Me!! + * + * @author Changyuan.lh + * @version 1.0 + */ +public class LogBuffer +{ + protected byte[] buffer; + + protected int origin, limit; + protected int position; + + protected LogBuffer() + { + } + + public LogBuffer(byte[] buffer, final int origin, final int limit) + { + if (origin + limit > buffer.length) + throw new IllegalArgumentException("capacity excceed: " + + (origin + limit)); + + this.buffer = buffer; + this.origin = origin; + this.position = origin; + this.limit = limit; + } + + /** + * Return n bytes in this buffer. + */ + public final LogBuffer duplicate(final int pos, final int len) + { + if (pos + len > limit) + throw new IllegalArgumentException("limit excceed: " + (pos + len)); + + // XXX: Do momery copy avoid buffer modified. + final int off = origin + pos; + byte[] buf = Arrays.copyOfRange(buffer, off, off + len); + return new LogBuffer(buf, 0, len); + } + + /** + * Return next n bytes in this buffer. + */ + public final LogBuffer duplicate(final int len) + { + if (position + len > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + // XXX: Do momery copy avoid buffer modified. + final int end = position + len; + byte[] buf = Arrays.copyOfRange(buffer, position, end); + LogBuffer dupBuffer = new LogBuffer(buf, 0, len); + position = end; + return dupBuffer; + } + + /** + * Return next n bytes in this buffer. + */ + public final LogBuffer duplicate() + { + // XXX: Do momery copy avoid buffer modified. + byte[] buf = Arrays.copyOfRange(buffer, origin, origin + limit); + return new LogBuffer(buf, 0, limit); + } + + /** + * Returns this buffer's capacity.

+ * + * @return The capacity of this buffer + */ + public final int capacity() + { + return buffer.length; + } + + /** + * Returns this buffer's position.

+ * + * @return The position of this buffer + */ + public final int position() + { + return position - origin; + } + + /** + * Sets this buffer's position. If the mark is defined and larger than the + * new position then it is discarded.

+ * + * @param newPosition The new position value; must be non-negative and no + * larger than the current limit + * + * @return This buffer + * + * @throws IllegalArgumentException If the preconditions on + * newPosition do not hold + */ + public final LogBuffer position(final int newPosition) + { + if (newPosition > limit || newPosition < 0) + throw new IllegalArgumentException("limit excceed: " + newPosition); + + this.position = origin + newPosition; + return this; + } + + /** + * Forwards this buffer's position. + * + * @param len The forward distance + * + * @return This buffer + */ + public final LogBuffer forward(final int len) + { + if (position + len > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + this.position += len; + return this; + } + + /** + * Consume this buffer, moving origin and position. + * + * @param len The consume distance + * + * @return This buffer + */ + public final LogBuffer consume(final int len) + { + if (limit > len) + { + limit -= len; + origin += len; + position = origin; + return this; + } + else if (limit == len) + { + limit = 0; + origin = 0; + position = 0; + return this; + } + else + { + /* Should not happen. */ + throw new IllegalArgumentException("limit excceed: " + len); + } + } + + /** + * Rewinds this buffer. The position is set to zero. + * + * @return This buffer + */ + public final LogBuffer rewind() + { + position = origin; + return this; + } + + /** + * Returns this buffer's limit.

+ * + * @return The limit of this buffer + */ + public final int limit() + { + return limit; + } + + /** + * Sets this buffer's limit. If the position is larger than the new limit + * then it is set to the new limit. If the mark is defined and larger than + * the new limit then it is discarded.

+ * + * @param newLimit The new limit value; must be non-negative and no larger + * than this buffer's capacity + * + * @return This buffer + * + * @throws IllegalArgumentException If the preconditions on + * newLimit do not hold + */ + public final LogBuffer limit(int newLimit) + { + if (origin + newLimit > buffer.length || newLimit < 0) + throw new IllegalArgumentException("capacity excceed: " + + (origin + newLimit)); + + limit = newLimit; + return this; + } + + /** + * Returns the number of elements between the current position and the + * limit.

+ * + * @return The number of elements remaining in this buffer + */ + public final int remaining() + { + return limit + origin - position; + } + + /** + * Tells whether there are any elements between the current position and the + * limit.

+ * + * @return true if, and only if, there is at least one element + * remaining in this buffer + */ + public final boolean hasRemaining() + { + return position < limit + origin; + } + + /** + * Return 8-bit signed int from buffer. + */ + public final int getInt8(final int pos) + { + if (pos >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + pos); + + return buffer[origin + pos]; + } + + /** + * Return next 8-bit signed int from buffer. + */ + public final int getInt8() + { + if (position >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin)); + + return buffer[position++]; + } + + /** + * Return 8-bit unsigned int from buffer. + */ + public final int getUint8(final int pos) + { + if (pos >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + pos); + + return 0xff & buffer[origin + pos]; + } + + /** + * Return next 8-bit unsigned int from buffer. + */ + public final int getUint8() + { + if (position >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin)); + + return 0xff & buffer[position++]; + } + + /** + * Return 16-bit signed int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint2korr + */ + public final int getInt16(final int pos) + { + final int position = origin + pos; + + if (pos + 1 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 1))); + + byte[] buf = buffer; + return (0xff & buf[position]) | ((buf[position + 1]) << 8); + } + + /** + * Return next 16-bit signed int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint2korr + */ + public final int getInt16() + { + if (position + 1 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 1)); + + byte[] buf = buffer; + return (0xff & buf[position++]) | ((buf[position++]) << 8); + } + + /** + * Return 16-bit unsigned int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint2korr + */ + public final int getUint16(final int pos) + { + final int position = origin + pos; + + if (pos + 1 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 1))); + + byte[] buf = buffer; + return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8); + } + + /** + * Return next 16-bit unsigned int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint2korr + */ + public final int getUint16() + { + if (position + 1 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 1)); + + byte[] buf = buffer; + return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8); + } + + /** + * Return 16-bit signed int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint2korr + */ + public final int getBeInt16(final int pos) + { + final int position = origin + pos; + + if (pos + 1 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 1))); + + byte[] buf = buffer; + return (0xff & buf[position + 1]) | ((buf[position]) << 8); + } + + /** + * Return next 16-bit signed int from buffer. (big-endian) + * + * @see mysql-5.1.60/include/my_global.h - mi_sint2korr + */ + public final int getBeInt16() + { + if (position + 1 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 1)); + + byte[] buf = buffer; + return (buf[position++] << 8) | (0xff & buf[position++]); + } + + /** + * Return 16-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint2korr + */ + public final int getBeUint16(final int pos) + { + final int position = origin + pos; + + if (pos + 1 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 1))); + + byte[] buf = buffer; + return (0xff & buf[position + 1]) | ((0xff & buf[position]) << 8); + } + + /** + * Return next 16-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint2korr + */ + public final int getBeUint16() + { + if (position + 1 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 1)); + + byte[] buf = buffer; + return ((0xff & buf[position++]) << 8) | (0xff & buf[position++]); + } + + /** + * Return 24-bit signed int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint3korr + */ + public final int getInt24(final int pos) + { + final int position = origin + pos; + + if (pos + 2 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 2))); + + byte[] buf = buffer; + return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8) + | ((buf[position + 2]) << 16); + } + + /** + * Return next 24-bit signed int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint3korr + */ + public final int getInt24() + { + if (position + 2 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 2)); + + byte[] buf = buffer; + return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8) + | ((buf[position++]) << 16); + } + + /** + * Return 24-bit signed int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr + */ + public final int getBeInt24(final int pos) + { + final int position = origin + pos; + + if (pos + 2 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 2))); + + byte[] buf = buffer; + return (0xff & buf[position + 2]) | ((0xff & buf[position + 1]) << 8) + | ((buf[position]) << 16); + } + + /** + * Return next 24-bit signed int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr + */ + public final int getBeInt24() + { + if (position + 2 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 2)); + + byte[] buf = buffer; + return ((buf[position++]) << 16) | ((0xff & buf[position++]) << 8) + | (0xff & buf[position++]); + } + + + /** + * Return 24-bit unsigned int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint3korr + */ + public final int getUint24(final int pos) + { + final int position = origin + pos; + + if (pos + 2 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 2))); + + byte[] buf = buffer; + return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8) + | ((0xff & buf[position + 2]) << 16); + } + + /** + * Return next 24-bit unsigned int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint3korr + */ + public final int getUint24() + { + if (position + 2 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 2)); + + byte[] buf = buffer; + return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8) + | ((0xff & buf[position++]) << 16); + } + + /** + * Return 24-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr + */ + public final int getBeUint24(final int pos) + { + final int position = origin + pos; + + if (pos + 2 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 2))); + + byte[] buf = buffer; + return (0xff & buf[position + 2]) | ((0xff & buf[position + 1]) << 8) + | ((0xff & buf[position]) << 16); + } + + /** + * Return next 24-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr + */ + public final int getBeUint24() + { + if (position + 2 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 2)); + + byte[] buf = buffer; + return ((0xff & buf[position++]) << 16) | ((0xff & buf[position++]) << 8) + | (0xff & buf[position++]); + } + + /** + * Return 32-bit signed int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint4korr + */ + public final int getInt32(final int pos) + { + final int position = origin + pos; + + if (pos + 3 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 3))); + + byte[] buf = buffer; + return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8) + | ((0xff & buf[position + 2]) << 16) + | ((buf[position + 3]) << 24); + } + + /** + * Return 32-bit signed int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint4korr + */ + public final int getBeInt32(final int pos) + { + final int position = origin + pos; + + if (pos + 3 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 3))); + + byte[] buf = buffer; + return (0xff & buf[position + 3]) | ((0xff & buf[position + 2]) << 8) + | ((0xff & buf[position + 1]) << 16) + | ((buf[position]) << 24); + } + + /** + * Return next 32-bit signed int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint4korr + */ + public final int getInt32() + { + if (position + 3 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 3)); + + byte[] buf = buffer; + return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8) + | ((0xff & buf[position++]) << 16) | ((buf[position++]) << 24); + } + + /** + * Return next 32-bit signed int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint4korr + */ + public final int getBeInt32() + { + if (position + 3 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 3)); + + byte[] buf = buffer; + return ((buf[position++]) << 24) | ((0xff & buf[position++]) << 16) | ((0xff & buf[position++]) << 8) + | (0xff & buf[position++]); + } + + /** + * Return 32-bit unsigned int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint4korr + */ + public final long getUint32(final int pos) + { + final int position = origin + pos; + + if (pos + 3 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 3))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position])) + | ((long) (0xff & buf[position + 1]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 3]) << 24); + } + + /** + * Return 32-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_usint4korr + */ + public final long getBeUint32(final int pos) + { + final int position = origin + pos; + + if (pos + 3 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 3))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position + 3])) + | ((long) (0xff & buf[position + 2]) << 8) + | ((long) (0xff & buf[position + 1]) << 16) + | ((long) (0xff & buf[position]) << 24); + } + + + /** + * Return next 32-bit unsigned int from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint4korr + */ + public final long getUint32() + { + if (position + 3 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 3)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++])) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 24); + } + + /** + * Return next 32-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint4korr + */ + public final long getBeUint32() + { + if (position + 3 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 3)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++])); + } + + /** + * Return 40-bit unsigned int from buffer. (little-endian) + */ + public final long getUlong40(final int pos) + { + final int position = origin + pos; + + if (pos + 4 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 4))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position])) + | ((long) (0xff & buf[position + 1]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 3]) << 24) + | ((long) (0xff & buf[position + 4]) << 32); + } + + /** + * Return next 40-bit unsigned int from buffer. (little-endian) + */ + public final long getUlong40() + { + if (position + 4 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 4)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++])) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 32); + } + + + /** + * Return 40-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint5korr + */ + public final long getBeUlong40(final int pos) + { + final int position = origin + pos; + + if (pos + 4 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 4))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position + 4])) + | ((long) (0xff & buf[position + 3]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 1]) << 24) + | ((long) (0xff & buf[position]) << 32); + } + + /** + * Return next 40-bit unsigned int from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint5korr + */ + public final long getBeUlong40() + { + if (position + 4 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 4)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++])); + } + + /** + * Return 48-bit signed long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint6korr + */ + public final long getLong48(final int pos) + { + final int position = origin + pos; + + if (pos + 5 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 5))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position])) + | ((long) (0xff & buf[position + 1]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 3]) << 24) + | ((long) (0xff & buf[position + 4]) << 32) + | ((long) (buf[position + 5]) << 40); + } + + /** + * Return 48-bit signed long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint6korr + */ + public final long getBeLong48(final int pos) + { + final int position = origin + pos; + + if (pos + 5 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 5))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position + 5])) + | ((long) (0xff & buf[position + 4]) << 8) + | ((long) (0xff & buf[position + 3]) << 16) + | ((long) (0xff & buf[position + 2]) << 24) + | ((long) (0xff & buf[position + 1]) << 32) + | ((long) (buf[position]) << 40); + } + + /** + * Return next 48-bit signed long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint6korr + */ + public final long getLong48() + { + if (position + 5 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 5)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++])) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (buf[position++]) << 40); + } + + /** + * Return next 48-bit signed long from buffer. (Big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint6korr + */ + public final long getBeLong48() + { + if (position + 5 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 5)); + + byte[] buf = buffer; + return ((long) (buf[position++]) << 40) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++])); + } + + /** + * Return 48-bit unsigned long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint6korr + */ + public final long getUlong48(final int pos) + { + final int position = origin + pos; + + if (pos + 5 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 5))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position])) + | ((long) (0xff & buf[position + 1]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 3]) << 24) + | ((long) (0xff & buf[position + 4]) << 32) + | ((long) (0xff & buf[position + 5]) << 40); + } + + /** + * Return 48-bit unsigned long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint6korr + */ + public final long getBeUlong48(final int pos) + { + final int position = origin + pos; + + if (pos + 5 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 5))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position + 5])) + | ((long) (0xff & buf[position + 4]) << 8) + | ((long) (0xff & buf[position + 3]) << 16) + | ((long) (0xff & buf[position + 2]) << 24) + | ((long) (0xff & buf[position + 1]) << 32) + | ((long) (0xff & buf[position]) << 40); + } + + /** + * Return next 48-bit unsigned long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint6korr + */ + public final long getUlong48() + { + if (position + 5 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 5)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++])) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 40); + } + + /** + * Return next 48-bit unsigned long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint6korr + */ + public final long getBeUlong48() + { + if (position + 5 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 5)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++]) << 40) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++])); + } + + /** + * Return 56-bit unsigned int from buffer. (little-endian) + * + */ + public final long getUlong56(final int pos) + { + final int position = origin + pos; + + if (pos + 6 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 6))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position])) + | ((long) (0xff & buf[position + 1]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 3]) << 24) + | ((long) (0xff & buf[position + 4]) << 32) + | ((long) (0xff & buf[position + 5]) << 40) + | ((long) (0xff & buf[position + 6]) << 48); + } + + /** + * Return next 56-bit unsigned int from buffer. (little-endian) + * + */ + public final long getUlong56() + { + if (position + 6 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 6)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++]) ) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 40) + | ((long) (0xff & buf[position++]) << 48); + } + + /** + * Return 56-bit unsigned int from buffer. (big-endian) + * + */ + public final long getBeUlong56(final int pos) + { + final int position = origin + pos; + + if (pos + 6 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 6))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position + 6])) + | ((long) (0xff & buf[position + 5]) << 8) + | ((long) (0xff & buf[position + 4]) << 16) + | ((long) (0xff & buf[position + 3]) << 24) + | ((long) (0xff & buf[position + 2]) << 32) + | ((long) (0xff & buf[position + 1]) << 40) + | ((long) (0xff & buf[position]) << 48); + } + + /** + * Return next 56-bit unsigned int from buffer. (big-endian) + * + */ + public final long getBeUlong56() + { + if (position + 6 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 6)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++]) << 48) + | ((long) (0xff & buf[position++]) << 40) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++])); + } + + /** + * Return 64-bit signed long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint8korr + */ + public final long getLong64(final int pos) + { + final int position = origin + pos; + + if (pos + 7 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 7))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position])) + | ((long) (0xff & buf[position + 1]) << 8) + | ((long) (0xff & buf[position + 2]) << 16) + | ((long) (0xff & buf[position + 3]) << 24) + | ((long) (0xff & buf[position + 4]) << 32) + | ((long) (0xff & buf[position + 5]) << 40) + | ((long) (0xff & buf[position + 6]) << 48) + | ((long) (buf[position + 7]) << 56); + } + + /** + * Return 64-bit signed long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint8korr + */ + public final long getBeLong64(final int pos) + { + final int position = origin + pos; + + if (pos + 7 >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + 7))); + + byte[] buf = buffer; + return ((long) (0xff & buf[position + 7])) + | ((long) (0xff & buf[position + 6]) << 8) + | ((long) (0xff & buf[position + 5]) << 16) + | ((long) (0xff & buf[position + 4]) << 24) + | ((long) (0xff & buf[position + 3]) << 32) + | ((long) (0xff & buf[position + 2]) << 40) + | ((long) (0xff & buf[position + 1]) << 48) + | ((long) (buf[position]) << 56); + } + + /** + * Return next 64-bit signed long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - sint8korr + */ + public final long getLong64() + { + if (position + 7 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 7)); + + byte[] buf = buffer; + return ((long) (0xff & buf[position++])) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 40) + | ((long) (0xff & buf[position++]) << 48) + | ((long) (buf[position++]) << 56); + } + + /** + * Return next 64-bit signed long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_sint8korr + */ + public final long getBeLong64() + { + if (position + 7 >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position - origin + 7)); + + byte[] buf = buffer; + return ((long) (buf[position++]) << 56) + | ((long) (0xff & buf[position++]) << 48) + | ((long) (0xff & buf[position++]) << 40) + | ((long) (0xff & buf[position++]) << 32) + | ((long) (0xff & buf[position++]) << 24) + | ((long) (0xff & buf[position++]) << 16) + | ((long) (0xff & buf[position++]) << 8) + | ((long) (0xff & buf[position++])); + } + + /* The max ulonglong - 0x ff ff ff ff ff ff ff ff */ + public static final BigInteger BIGINT_MAX_VALUE = new BigInteger( + "18446744073709551615"); + + /** + * Return 64-bit unsigned long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint8korr + */ + public final BigInteger getUlong64(final int pos) + { + final long long64 = getLong64(pos); + + return (long64 >= 0) ? BigInteger.valueOf(long64) + : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64)); + } + + /** + * Return 64-bit unsigned long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint8korr + */ + public final BigInteger getBeUlong64(final int pos) + { + final long long64 = getBeLong64(pos); + + return (long64 >= 0) ? BigInteger.valueOf(long64) + : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64)); + } + + /** + * Return next 64-bit unsigned long from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - uint8korr + */ + public final BigInteger getUlong64() + { + final long long64 = getLong64(); + + return (long64 >= 0) ? BigInteger.valueOf(long64) + : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64)); + } + + /** + * Return next 64-bit unsigned long from buffer. (big-endian) + * + * @see mysql-5.6.10/include/myisampack.h - mi_uint8korr + */ + public final BigInteger getBeUlong64() + { + final long long64 = getBeLong64(); + + return (long64 >= 0) ? BigInteger.valueOf(long64) + : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64)); + } + + /** + * Return 32-bit float from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - float4get + */ + public final float getFloat32(final int pos) + { + return Float.intBitsToFloat(getInt32(pos)); + } + + /** + * Return next 32-bit float from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - float4get + */ + public final float getFloat32() + { + return Float.intBitsToFloat(getInt32()); + } + + /** + * Return 64-bit double from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - float8get + */ + public final double getDouble64(final int pos) + { + return Double.longBitsToDouble(getLong64(pos)); + } + + /** + * Return next 64-bit double from buffer. (little-endian) + * + * @see mysql-5.1.60/include/my_global.h - float8get + */ + public final double getDouble64() + { + return Double.longBitsToDouble(getLong64()); + } + + public static final long NULL_LENGTH = ((long) ~0); + + /** + * Return packed number from buffer. (little-endian) + * + * A Packed Integer has the capacity of storing up to 8-byte integers, while + * small integers still can use 1, 3, or 4 bytes. The value of the first + * byte determines how to read the number, according to the following table. + * + *
    + *
  • 0-250 The first byte is the number (in the range 0-250). No + * additional bytes are used.
    • + *
  • 252 Two more bytes are used. The number is in the range 251-0xffff.
    • + *
  • 253 Three more bytes are used. The number is in the range + * 0xffff-0xffffff.
    • + *
  • 254 Eight more bytes are used. The number is in the range + * 0xffffff-0xffffffffffffffff.
    • + *
+ * + * That representation allows a first byte value of 251 to represent the SQL + * NULL value. + */ + public final long getPackedLong(final int pos) + { + final int lead = getUint8(pos); + if (lead < 251) + return lead; + + switch (lead) + { + case 251: + return NULL_LENGTH; + case 252: + return getUint16(pos + 1); + case 253: + return getUint24(pos + 1); + default: /* Must be 254 when here */ + return getUint32(pos + 1); + } + } + + /** + * Return next packed number from buffer. (little-endian) + * + * @see LogBuffer#getPackedLong(int) + */ + public final long getPackedLong() + { + final int lead = getUint8(); + if (lead < 251) + return lead; + + switch (lead) + { + case 251: + return NULL_LENGTH; + case 252: + return getUint16(); + case 253: + return getUint24(); + default: /* Must be 254 when here */ + final long value = getUint32(); + position += 4; /* ignore other */ + return value; + } + } + + /* default ANSI charset */ + public static final String ISO_8859_1 = "ISO-8859-1"; + + /** + * Return fix length string from buffer. + */ + public final String getFixString(final int pos, final int len) + { + return getFixString(pos, len, ISO_8859_1); + } + + /** + * Return next fix length string from buffer. + */ + public final String getFixString(final int len) + { + return getFixString(len, ISO_8859_1); + } + + /** + * Return fix length string from buffer. + */ + public final String getFixString(final int pos, final int len, + String charsetName) + { + if (pos + len > limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + len))); + + final int from = origin + pos; + final int end = from + len; + byte[] buf = buffer; + int found = from; + for (; (found < end) && buf[found] != '\0'; found++) + /* empty loop */; + + try + { + return new String(buf, from, found - from, charsetName); + } + catch (UnsupportedEncodingException e) + { + throw new IllegalArgumentException("Unsupported encoding: " + + charsetName, e); + } + } + + /** + * Return next fix length string from buffer. + */ + public final String getFixString(final int len, String charsetName) + { + if (position + len > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + final int from = position; + final int end = from + len; + byte[] buf = buffer; + int found = from; + for (; (found < end) && buf[found] != '\0'; found++) + /* empty loop */; + + try + { + String string = new String(buf, from, found - from, charsetName); + position += len; + return string; + } + catch (UnsupportedEncodingException e) + { + throw new IllegalArgumentException("Unsupported encoding: " + + charsetName, e); + } + } + + /** + * Return fix-length string from buffer without null-terminate checking. + * + * Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 } + */ + public final String getFullString(final int pos, final int len, + String charsetName) + { + if (pos + len > limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + len))); + + try + { + return new String(buffer, origin + pos, len, charsetName); + } + catch (UnsupportedEncodingException e) + { + throw new IllegalArgumentException("Unsupported encoding: " + + charsetName, e); + } + } + + /** + * Return next fix-length string from buffer without null-terminate + * checking. + * + * Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 } + */ + public final String getFullString(final int len, String charsetName) + { + if (position + len > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + try + { + String string = new String(buffer, position, len, charsetName); + position += len; + return string; + } + catch (UnsupportedEncodingException e) + { + throw new IllegalArgumentException("Unsupported encoding: " + + charsetName, e); + } + } + + /** + * Return dynamic length string from buffer. + */ + public final String getString(final int pos) + { + return getString(pos, ISO_8859_1); + } + + /** + * Return next dynamic length string from buffer. + */ + public final String getString() + { + return getString(ISO_8859_1); + } + + /** + * Return dynamic length string from buffer. + */ + public final String getString(final int pos, String charsetName) + { + if (pos >= limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + pos); + + byte[] buf = buffer; + final int len = (0xff & buf[origin + pos]); + if (pos + len + 1 > limit) + throw new IllegalArgumentException("limit excceed: " + + (pos + len + 1)); + + try + { + return new String(buf, origin + pos + 1, len, charsetName); + } + catch (UnsupportedEncodingException e) + { + throw new IllegalArgumentException("Unsupported encoding: " + + charsetName, e); + } + } + + /** + * Return next dynamic length string from buffer. + */ + public final String getString(String charsetName) + { + if (position >= origin + limit) + throw new IllegalArgumentException("limit excceed: " + position); + + byte[] buf = buffer; + final int len = (0xff & buf[position]); + if (position + len + 1 > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len + 1 - origin)); + + try + { + String string = new String(buf, position + 1, len, charsetName); + position += len + 1; + return string; + } + catch (UnsupportedEncodingException e) + { + throw new IllegalArgumentException("Unsupported encoding: " + + charsetName, e); + } + } + + /** + * Return 16-bit signed int from buffer. (big-endian) + * + * @see mysql-5.1.60/include/myisampack.h - mi_sint2korr + */ + private static final int getInt16BE(byte[] buffer, final int pos) + { + return ((buffer[pos]) << 8) | (0xff & buffer[pos + 1]); + } + + /** + * Return 24-bit signed int from buffer. (big-endian) + * + * @see mysql-5.1.60/include/myisampack.h - mi_sint3korr + */ + private static final int getInt24BE(byte[] buffer, final int pos) + { + return (buffer[pos] << 16) | ((0xff & buffer[pos + 1]) << 8) + | (0xff & buffer[pos + 2]); + } + + /** + * Return 32-bit signed int from buffer. (big-endian) + * + * @see mysql-5.1.60/include/myisampack.h - mi_sint4korr + */ + private static final int getInt32BE(byte[] buffer, final int pos) + { + return (buffer[pos] << 24) | ((0xff & buffer[pos + 1]) << 16) + | ((0xff & buffer[pos + 2]) << 8) | (0xff & buffer[pos + 3]); + } + + /* decimal representation */ + public static final int DIG_PER_DEC1 = 9; + public static final int DIG_BASE = 1000000000; + public static final int DIG_MAX = DIG_BASE - 1; + public static final int dig2bytes[] = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4 }; + public static final int powers10[] = { 1, 10, 100, 1000, 10000, 100000, + 1000000, 10000000, 100000000, 1000000000 }; + + public static final int DIG_PER_INT32 = 9; + public static final int SIZE_OF_INT32 = 4; + + /** + * Return big decimal from buffer. + * + * @see mysql-5.1.60/strings/decimal.c - bin2decimal() + */ + public final BigDecimal getDecimal(final int pos, final int precision, + final int scale) + { + final int intg = precision - scale; + final int frac = scale; + final int intg0 = intg / DIG_PER_INT32; + final int frac0 = frac / DIG_PER_INT32; + final int intg0x = intg - intg0 * DIG_PER_INT32; + final int frac0x = frac - frac0 * DIG_PER_INT32; + + final int binSize = intg0 * SIZE_OF_INT32 + dig2bytes[intg0x] + frac0 + * SIZE_OF_INT32 + dig2bytes[frac0x]; + if (pos + binSize > limit || pos < 0) + { + throw new IllegalArgumentException("limit excceed: " + + (pos < 0 ? pos : (pos + binSize))); + } + return getDecimal0(origin + pos, intg, frac, // NL + intg0, frac0, intg0x, frac0x); + } + + /** + * Return next big decimal from buffer. + * + * @see mysql-5.1.60/strings/decimal.c - bin2decimal() + */ + public final BigDecimal getDecimal(final int precision, final int scale) + { + final int intg = precision - scale; + final int frac = scale; + final int intg0 = intg / DIG_PER_INT32; + final int frac0 = frac / DIG_PER_INT32; + final int intg0x = intg - intg0 * DIG_PER_INT32; + final int frac0x = frac - frac0 * DIG_PER_INT32; + + final int binSize = intg0 * SIZE_OF_INT32 + dig2bytes[intg0x] + frac0 + * SIZE_OF_INT32 + dig2bytes[frac0x]; + if (position + binSize > origin + limit) + { + throw new IllegalArgumentException("limit excceed: " + + (position + binSize - origin)); + } + + BigDecimal decimal = getDecimal0(position, intg, frac, // NL + intg0, frac0, intg0x, frac0x); + position += binSize; + return decimal; + } + + /** + * Return big decimal from buffer. + * + * 
+     * Decimal representation in binlog seems to be as follows:
+     * 
+     * 1st bit - sign such that set == +, unset == -
+     * every 4 bytes represent 9 digits in big-endian order, so that
+     * if you print the values of these quads as big-endian integers one after
+     * another, you get the whole number string representation in decimal. What
+     * remains is to put a sign and a decimal dot.
+     * 
+     * 80 00 00 05 1b 38 b0 60 00 means:
+     * 
+     *   0x80 - positive 
+     *   0x00000005 - 5
+     *   0x1b38b060 - 456700000
+     *   0x00       - 0
+     * 
+     * 54567000000 / 10^{10} = 5.4567
+     *
+ * + * @see mysql-5.1.60/strings/decimal.c - bin2decimal() + * @see mysql-5.1.60/strings/decimal.c - decimal2string() + */ + private final BigDecimal getDecimal0(final int begin, final int intg, + final int frac, final int intg0, final int frac0, final int intg0x, + final int frac0x) + { + final int mask = ((buffer[begin] & 0x80) == 0x80) ? 0 : -1; + int from = begin; + + /* max string length */ + final int len = ((mask != 0) ? 1 : 0) + ((intg != 0) ? intg : 1) // NL + + ((frac != 0) ? 1 : 0) + frac; + char[] buf = new char[len]; + int pos = 0; + + if (mask != 0) /* decimal sign */ + buf[pos++] = ('-'); + + final byte[] d_copy = buffer; + d_copy[begin] ^= 0x80; /* clear sign */ + int mark = pos; + + if (intg0x != 0) + { + final int i = dig2bytes[intg0x]; + int x = 0; + switch (i) + { + case 1: + x = d_copy[from] /* one byte */; + break; + case 2: + x = getInt16BE(d_copy, from); + break; + case 3: + x = getInt24BE(d_copy, from); + break; + case 4: + x = getInt32BE(d_copy, from); + break; + } + from += i; + x ^= mask; + if (x < 0 || x >= powers10[intg0x + 1]) + { + throw new IllegalArgumentException("bad format, x exceed: " + x + + ", " + powers10[intg0x + 1]); + } + if (x != 0 /* !digit || x != 0 */) + { + for (int j = intg0x; j > 0; j--) + { + final int divisor = powers10[j - 1]; + final int y = x / divisor; + if (mark < pos || y != 0) + { + buf[pos++] = ((char) ('0' + y)); + } + x -= y * divisor; + } + } + } + + for (final int stop = from + intg0 * SIZE_OF_INT32; from < stop; from += SIZE_OF_INT32) + { + int x = getInt32BE(d_copy, from); + x ^= mask; + if (x < 0 || x > DIG_MAX) + { + throw new IllegalArgumentException("bad format, x exceed: " + x + + ", " + DIG_MAX); + } + if (x != 0) + { + if (mark < pos) + { + for (int i = DIG_PER_DEC1; i > 0; i--) + { + final int divisor = powers10[i - 1]; + final int y = x / divisor; + buf[pos++] = ((char) ('0' + y)); + x -= y * divisor; + } + } + else + { + for (int i = DIG_PER_DEC1; i > 0; i--) + { + final int divisor = powers10[i - 1]; + final int y = x / divisor; + if (mark < pos || y != 0) + { + buf[pos++] = ((char) ('0' + y)); + } + x -= y * divisor; + } + } + } + else if (mark < pos) + { + for (int i = DIG_PER_DEC1; i > 0; i--) + buf[pos++] = ('0'); + } + } + + if (mark == pos) + /* fix 0.0 problem, only '.' may cause BigDecimal parsing exception. */ + buf[pos++] = ('0'); + + if (frac > 0) + { + buf[pos++] = ('.'); + mark = pos; + + for (final int stop = from + frac0 * SIZE_OF_INT32; from < stop; from += SIZE_OF_INT32) + { + int x = getInt32BE(d_copy, from); + x ^= mask; + if (x < 0 || x > DIG_MAX) + { + throw new IllegalArgumentException("bad format, x exceed: " + + x + ", " + DIG_MAX); + } + if (x != 0) + { + for (int i = DIG_PER_DEC1; i > 0; i--) + { + final int divisor = powers10[i - 1]; + final int y = x / divisor; + buf[pos++] = ((char) ('0' + y)); + x -= y * divisor; + } + } + else + { + for (int i = DIG_PER_DEC1; i > 0; i--) + buf[pos++] = ('0'); + } + } + + if (frac0x != 0) + { + final int i = dig2bytes[frac0x]; + int x = 0; + switch (i) + { + case 1: + x = d_copy[from] /* one byte */; + break; + case 2: + x = getInt16BE(d_copy, from); + break; + case 3: + x = getInt24BE(d_copy, from); + break; + case 4: + x = getInt32BE(d_copy, from); + break; + } + x ^= mask; + if (x != 0) + { + final int dig = DIG_PER_DEC1 - frac0x; + x *= powers10[dig]; + if (x < 0 || x > DIG_MAX) + { + throw new IllegalArgumentException( + "bad format, x exceed: " + x + ", " + DIG_MAX); + } + for (int j = DIG_PER_DEC1; j > dig; j--) + { + final int divisor = powers10[j - 1]; + final int y = x / divisor; + buf[pos++] = ((char) ('0' + y)); + x -= y * divisor; + } + } + } + + if (mark == pos) + /* make number more friendly */ + buf[pos++] = ('0'); + } + + d_copy[begin] ^= 0x80; /* restore sign */ + String decimal = String.valueOf(buf, 0, pos); + return new BigDecimal(decimal); + } + + /** + * Fill MY_BITMAP structure from buffer. + * + * @param len The length of MY_BITMAP in bits. + */ + public final void fillBitmap(BitSet bitmap, final int pos, final int len) + { + if (pos + ((len + 7) / 8) > limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + + (pos + (len + 7) / 8)); + + fillBitmap0(bitmap, origin + pos, len); + } + + /** + * Fill next MY_BITMAP structure from buffer. + * + * @param len The length of MY_BITMAP in bits. + */ + public final void fillBitmap(BitSet bitmap, final int len) + { + if (position + ((len + 7) / 8) > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + ((len + 7) / 8) - origin)); + + position = fillBitmap0(bitmap, position, len); + } + + /** + * Fill MY_BITMAP structure from buffer. + * + * @param len The length of MY_BITMAP in bits. + */ + private final int fillBitmap0(BitSet bitmap, int pos, final int len) + { + final byte[] buf = buffer; + + for (int bit = 0; bit < len; bit += 8) + { + int flag = ((int) buf[pos++]) & 0xff; + if (flag == 0) + continue; + if ((flag & 0x01) != 0) + bitmap.set(bit); + if ((flag & 0x02) != 0) + bitmap.set(bit + 1); + if ((flag & 0x04) != 0) + bitmap.set(bit + 2); + if ((flag & 0x08) != 0) + bitmap.set(bit + 3); + if ((flag & 0x10) != 0) + bitmap.set(bit + 4); + if ((flag & 0x20) != 0) + bitmap.set(bit + 5); + if ((flag & 0x40) != 0) + bitmap.set(bit + 6); + if ((flag & 0x80) != 0) + bitmap.set(bit + 7); + } + return pos; + } + + /** + * Return MY_BITMAP structure from buffer. + * + * @param len The length of MY_BITMAP in bits. + */ + public final BitSet getBitmap(final int pos, final int len) + { + BitSet bitmap = new BitSet(len); + fillBitmap(bitmap, pos, len); + return bitmap; + } + + /** + * Return next MY_BITMAP structure from buffer. + * + * @param len The length of MY_BITMAP in bits. + */ + public final BitSet getBitmap(final int len) + { + BitSet bitmap = new BitSet(len); + fillBitmap(bitmap, len); + return bitmap; + } + + /** + * Fill n bytes into output stream. + */ + public final void fillOutput(OutputStream out, final int pos, final int len) + throws IOException + { + if (pos + len > limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + (pos + len)); + + out.write(buffer, origin + pos, len); + } + + /** + * Fill next n bytes into output stream. + */ + public final void fillOutput(OutputStream out, final int len) + throws IOException + { + if (position + len > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + out.write(buffer, position, len); + position += len; + } + + /** + * Fill n bytes in this buffer. + */ + public final void fillBytes(final int pos, byte[] dest, final int destPos, + final int len) + { + if (pos + len > limit || pos < 0) + throw new IllegalArgumentException("limit excceed: " + (pos + len)); + + System.arraycopy(buffer, origin + pos, dest, destPos, len); + } + + /** + * Fill next n bytes in this buffer. + */ + public final void fillBytes(byte[] dest, final int destPos, final int len) + { + if (position + len > origin + limit) + throw new IllegalArgumentException("limit excceed: " + + (position + len - origin)); + + System.arraycopy(buffer, position, dest, destPos, len); + position += len; + } + + /** + * Return n-byte data from buffer. + */ + public final byte[] getData(final int pos, final int len) + { + byte[] buf = new byte[len]; + fillBytes(pos, buf, 0, len); + return buf; + } + + /** + * Return next n-byte data from buffer. + */ + public final byte[] getData(final int len) + { + byte[] buf = new byte[len]; + fillBytes(buf, 0, len); + return buf; + } + + /** + * Return all remaining data from buffer. + */ + public final byte[] getData() + { + return getData(0, limit); + } + + /** + * Return full hexdump from position. + */ + public final String hexdump(final int pos) + { + if ((limit - pos) > 0) + { + final int begin = origin + pos; + final int end = origin + limit; + + byte[] buf = buffer; + StringBuilder dump = new StringBuilder(); + dump.append(Integer.toHexString(buf[begin] >> 4)); + dump.append(Integer.toHexString(buf[begin] & 0xf)); + for (int i = begin + 1; i < end; i++) + { + dump.append("_"); + dump.append(Integer.toHexString(buf[i] >> 4)); + dump.append(Integer.toHexString(buf[i] & 0xf)); + } + + return dump.toString(); + } + return ""; + } + + /** + * Return hexdump from position, for len bytes. + */ + public final String hexdump(final int pos, final int len) + { + if ((limit - pos) > 0) + { + final int begin = origin + pos; + final int end = Math.min(begin + len, origin + limit); + + byte[] buf = buffer; + StringBuilder dump = new StringBuilder(); + dump.append(Integer.toHexString(buf[begin] >> 4)); + dump.append(Integer.toHexString(buf[begin] & 0xf)); + for (int i = begin + 1; i < end; i++) + { + dump.append("_"); + dump.append(Integer.toHexString(buf[i] >> 4)); + dump.append(Integer.toHexString(buf[i] & 0xf)); + } + + return dump.toString(); + } + return ""; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java new file mode 100644 index 00000000..0a1f04e2 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java @@ -0,0 +1,77 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.util.HashMap; +import java.util.Map; + +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent; + +/** + * TODO: Document Me!! + * + * NOTE: Log context will NOT write multi-threaded. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class LogContext +{ + private final Map mapOfTable = new HashMap(); + + private FormatDescriptionLogEvent formatDescription; + + private LogPosition logPosition; + + public LogContext() + { + this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x; + } + + public LogContext(FormatDescriptionLogEvent descriptionEvent) + { + this.formatDescription = descriptionEvent; + } + + public final LogPosition getLogPosition() + { + return logPosition; + } + + public final void setLogPosition(LogPosition logPosition) + { + this.logPosition = logPosition; + } + + public final FormatDescriptionLogEvent getFormatDescription() + { + return formatDescription; + } + + public final void setFormatDescription( + FormatDescriptionLogEvent formatDescription) + { + this.formatDescription = formatDescription; + } + + public final void putTable(TableMapLogEvent mapEvent) + { + mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent); + } + + public final TableMapLogEvent getTable(final long tableId) + { + return mapOfTable.get(Long.valueOf(tableId)); + } + + public final void clearAllTables() + { + mapOfTable.clear(); + } + + public void reset() + { + formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x; + + mapOfTable.clear(); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java new file mode 100644 index 00000000..be86cc8a --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java @@ -0,0 +1,494 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.IOException; +import java.util.BitSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.taobao.tddl.dbsync.binlog.event.AppendBlockLogEvent; +import com.taobao.tddl.dbsync.binlog.event.BeginLoadQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.CreateFileLogEvent; +import com.taobao.tddl.dbsync.binlog.event.DeleteFileLogEvent; +import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.ExecuteLoadLogEvent; +import com.taobao.tddl.dbsync.binlog.event.ExecuteLoadQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent; +import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IncidentLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent; +import com.taobao.tddl.dbsync.binlog.event.LoadLogEvent; +import com.taobao.tddl.dbsync.binlog.event.LogHeader; +import com.taobao.tddl.dbsync.binlog.event.PreviousGtidsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RandLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.StartLogEventV3; +import com.taobao.tddl.dbsync.binlog.event.StopLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent; +import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent; +import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent; +import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.XidLogEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent; + +/** + * Implements a binary-log decoder. + * + * 
+ * LogDecoder decoder = new LogDecoder();
+ * decoder.handle(...);
+ * 
+ * LogEvent event;
+ * do
+ * {
+ *     event = decoder.decode(buffer, context);
+ * 
+ *     // process log event.
+ * }
+ * while (event != null);
+ * // no more events in buffer.
+ *
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class LogDecoder +{ + protected static final Log logger = LogFactory.getLog(LogDecoder.class); + + protected final BitSet handleSet = new BitSet(LogEvent.ENUM_END_EVENT); + + public LogDecoder() + { + } + + public LogDecoder(final int fromIndex, final int toIndex) + { + handleSet.set(fromIndex, toIndex); + } + + public final void handle(final int fromIndex, final int toIndex) + { + handleSet.set(fromIndex, toIndex); + } + + public final void handle(final int flagIndex) + { + handleSet.set(flagIndex); + } + + /** + * Decoding an event from binary-log buffer. + * + * @return UknownLogEvent if event type is unknown or skipped, + * null if buffer is not including a full event. + */ + public LogEvent decode(LogBuffer buffer, LogContext context) + throws IOException + { + final int limit = buffer.limit(); + + if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN) + { + LogHeader header = new LogHeader(buffer, + context.getFormatDescription()); + + final int len = header.getEventLen(); + if (limit >= len) + { + LogEvent event; + + /* Checking binary-log's header */ + if (handleSet.get(header.getType())) + { + buffer.limit(len); + try + { + /* Decoding binary-log to event */ + event = decode(buffer, header, context); + } + catch (IOException e) + { + if (logger.isWarnEnabled()) + logger.warn("Decoding " + + LogEvent.getTypeName(header.getType()) + + " failed from: " + + context.getLogPosition(), e); + throw e; + } + finally + { + buffer.limit(limit); /* Restore limit */ + } + } + else + { + /* Ignore unsupported binary-log. */ + event = new UnknownLogEvent(header); + } + + /* consume this binary-log. */ + buffer.consume(len); + return event; + } + } + + /* Rewind buffer's position to 0. */ + buffer.rewind(); + return null; + } + + /** + * Deserialize an event from buffer. + * + * @return UknownLogEvent if event type is unknown or skipped. + */ + public static LogEvent decode(LogBuffer buffer, LogHeader header, + LogContext context) throws IOException + { + FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription(); + LogPosition logPosition = context.getLogPosition(); + + int checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF; + if (header.getType() != LogEvent.FORMAT_DESCRIPTION_EVENT) { + checksumAlg = descriptionEvent.header.getChecksumAlg(); + }else { + // 如果是format事件自己，也需要处理checksum + checksumAlg = header.getChecksumAlg(); + } + + if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) { + // remove checksum bytes + buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN); + } + + switch (header.getType()) + { + case LogEvent.QUERY_EVENT: + { + QueryLogEvent event = new QueryLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.XID_EVENT: + { + XidLogEvent event = new XidLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.TABLE_MAP_EVENT: + { + TableMapLogEvent mapEvent = new TableMapLogEvent(header, + buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + context.putTable(mapEvent); + return mapEvent; + } + case LogEvent.WRITE_ROWS_EVENT_V1: + { + RowsLogEvent event = new WriteRowsLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + return event; + } + case LogEvent.UPDATE_ROWS_EVENT_V1: + { + RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + return event; + } + case LogEvent.DELETE_ROWS_EVENT_V1: + { + RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + return event; + } + case LogEvent.ROTATE_EVENT: + { + RotateLogEvent event = new RotateLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition = new LogPosition(event.getFilename(), + event.getPosition()); + context.setLogPosition(logPosition); + return event; + } + case LogEvent.LOAD_EVENT: + case LogEvent.NEW_LOAD_EVENT: + { + LoadLogEvent event = new LoadLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */ + { + if (logger.isWarnEnabled()) + logger.warn("Skipping unsupported SLAVE_EVENT from: " + + context.getLogPosition()); + break; + } + case LogEvent.CREATE_FILE_EVENT: + { + CreateFileLogEvent event = new CreateFileLogEvent(header, + buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.APPEND_BLOCK_EVENT: + { + AppendBlockLogEvent event = new AppendBlockLogEvent(header, + buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.DELETE_FILE_EVENT: + { + DeleteFileLogEvent event = new DeleteFileLogEvent(header, + buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.EXEC_LOAD_EVENT: + { + ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header, + buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.START_EVENT_V3: + { + /* This is sent only by MySQL <=4.x */ + StartLogEventV3 event = new StartLogEventV3(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.STOP_EVENT: + { + StopLogEvent event = new StopLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.INTVAR_EVENT: + { + IntvarLogEvent event = new IntvarLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.RAND_EVENT: + { + RandLogEvent event = new RandLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.USER_VAR_EVENT: + { + UserVarLogEvent event = new UserVarLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.FORMAT_DESCRIPTION_EVENT: + { + descriptionEvent = new FormatDescriptionLogEvent(header, + buffer, descriptionEvent); + context.setFormatDescription(descriptionEvent); + return descriptionEvent; + } + case LogEvent.PRE_GA_WRITE_ROWS_EVENT: + { + if (logger.isWarnEnabled()) + logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: " + + context.getLogPosition()); + // ev = new Write_rows_log_event_old(buf, event_len, + // description_event); + break; + } + case LogEvent.PRE_GA_UPDATE_ROWS_EVENT: + { + if (logger.isWarnEnabled()) + logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: " + + context.getLogPosition()); + // ev = new Update_rows_log_event_old(buf, event_len, + // description_event); + break; + } + case LogEvent.PRE_GA_DELETE_ROWS_EVENT: + { + if (logger.isWarnEnabled()) + logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: " + + context.getLogPosition()); + // ev = new Delete_rows_log_event_old(buf, event_len, + // description_event); + break; + } + case LogEvent.BEGIN_LOAD_QUERY_EVENT: + { + BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent( + header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.EXECUTE_LOAD_QUERY_EVENT: + { + ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent( + header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.INCIDENT_EVENT: + { + IncidentLogEvent event = new IncidentLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.HEARTBEAT_LOG_EVENT: + { + HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer, + descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.IGNORABLE_LOG_EVENT: + { + IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.ROWS_QUERY_LOG_EVENT: + { + RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.WRITE_ROWS_EVENT: { + RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + return event; + } + case LogEvent.UPDATE_ROWS_EVENT: { + RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + return event; + } + case LogEvent.DELETE_ROWS_EVENT: { + RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + event.fillTable(context); + return event; + } + case LogEvent.GTID_LOG_EVENT: + case LogEvent.ANONYMOUS_GTID_LOG_EVENT: + { + GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.PREVIOUS_GTIDS_LOG_EVENT: + { + PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.ANNOTATE_ROWS_EVENT: + { + AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.BINLOG_CHECKPOINT_EVENT: + { + BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.GTID_EVENT: + { + MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + case LogEvent.GTID_LIST_EVENT: + { + MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + } + default: + /* + Create an object of Ignorable_log_event for unrecognized sub-class. + So that SLAVE SQL THREAD will only update the position and continue. + */ + if((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0){ + IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent); + /* updating position in context */ + logPosition.position = header.getLogPos(); + return event; + }else { + if (logger.isWarnEnabled()) + logger.warn("Skipping unrecognized binlog event " + + LogEvent.getTypeName(header.getType()) + " from: " + + context.getLogPosition()); + } + } + + /* updating position in context */ + logPosition.position = header.getLogPos(); + /* Unknown or unsupported log event */ + return new UnknownLogEvent(header); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java new file mode 100644 index 00000000..7f9c3a8f --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java @@ -0,0 +1,437 @@ +package com.taobao.tddl.dbsync.binlog; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.taobao.tddl.dbsync.binlog.event.LogHeader; + +/** + * Binary log event definitions. This includes generic code common to all types + * of log events, as well as specific code for each type of log event. + * + * - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored + * in little endian, i.e., the least significant byte first, unless otherwise + * specified. + * + * representation of unsigned integers, called Packed Integer. A Packed Integer + * has the capacity of storing up to 8-byte integers, while small integers still + * can use 1, 3, or 4 bytes. The value of the first byte determines how to read + * the number, according to the following table: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Format of Packed Integer
First byteFormat
0-250The first byte is the number (in the range 0-250), and no more bytes are + * used.
252Two more bytes are used. The number is in the range 251-0xffff.
253Three more bytes are used. The number is in the range 0xffff-0xffffff.
254Eight more bytes are used. The number is in the range + * 0xffffff-0xffffffffffffffff.
+ * + * - Strings are stored in various formats. The format of each string is + * documented separately. + * + * @see mysql-5.1.60/sql/log_event.h + * + * @author Changyuan.lh + * @version 1.0 + */ +public abstract class LogEvent +{ + /* + * 3 is MySQL 4.x; 4 is MySQL 5.0.0. + * Compared to version 3, version 4 has: + * - a different Start_log_event, which includes info about the binary log + * (sizes of headers); this info is included for better compatibility if the + * master's MySQL version is different from the slave's. + * - all events have a unique ID (the triplet (server_id, timestamp at server + * start, other) to be sure an event is not executed more than once in a + * multimaster setup, example: + * M1 + * / \ + * v v + * M2 M3 + * \ / + * v v + * S + * if a query is run on M1, it will arrive twice on S, so we need that S + * remembers the last unique ID it has processed, to compare and know if the + * event should be skipped or not. Example of ID: we already have the server id + * (4 bytes), plus: + * timestamp_when_the_master_started (4 bytes), a counter (a sequence number + * which increments every time we write an event to the binlog) (3 bytes). + * Q: how do we handle when the counter is overflowed and restarts from 0 ? + * + * - Query and Load (Create or Execute) events may have a more precise + * timestamp (with microseconds), number of matched/affected/warnings rows + * and fields of session variables: SQL_MODE, + * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and + * charsets, the PASSWORD() version (old/new/...). + */ + public static final int BINLOG_VERSION = 4; + + /* Default 5.0 server version */ + public static final String SERVER_VERSION = "5.0"; + + /** + * Event header offsets; these point to places inside the fixed header. + */ + public static final int EVENT_TYPE_OFFSET = 4; + public static final int SERVER_ID_OFFSET = 5; + public static final int EVENT_LEN_OFFSET = 9; + public static final int LOG_POS_OFFSET = 13; + public static final int FLAGS_OFFSET = 17; + + /* event-specific post-header sizes */ + // where 3.23, 4.x and 5.0 agree + public static final int QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2); + // where 5.0 differs: 2 for len of N-bytes vars. + public static final int QUERY_HEADER_LEN = (QUERY_HEADER_MINIMAL_LEN + 2); + + /* Enumeration type for the different types of log events. */ + public static final int UNKNOWN_EVENT = 0; + public static final int START_EVENT_V3 = 1; + public static final int QUERY_EVENT = 2; + public static final int STOP_EVENT = 3; + public static final int ROTATE_EVENT = 4; + public static final int INTVAR_EVENT = 5; + public static final int LOAD_EVENT = 6; + public static final int SLAVE_EVENT = 7; + public static final int CREATE_FILE_EVENT = 8; + public static final int APPEND_BLOCK_EVENT = 9; + public static final int EXEC_LOAD_EVENT = 10; + public static final int DELETE_FILE_EVENT = 11; + + /** + * NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer sql_ex, + * allowing multibyte TERMINATED BY etc; both types share the same class + * (Load_log_event) + */ + public static final int NEW_LOAD_EVENT = 12; + public static final int RAND_EVENT = 13; + public static final int USER_VAR_EVENT = 14; + public static final int FORMAT_DESCRIPTION_EVENT = 15; + public static final int XID_EVENT = 16; + public static final int BEGIN_LOAD_QUERY_EVENT = 17; + public static final int EXECUTE_LOAD_QUERY_EVENT = 18; + + public static final int TABLE_MAP_EVENT = 19; + + /** + * These event numbers were used for 5.1.0 to 5.1.15 and are therefore + * obsolete. + */ + public static final int PRE_GA_WRITE_ROWS_EVENT = 20; + public static final int PRE_GA_UPDATE_ROWS_EVENT = 21; + public static final int PRE_GA_DELETE_ROWS_EVENT = 22; + + /** + * These event numbers are used from 5.1.16 and forward + */ + public static final int WRITE_ROWS_EVENT_V1 = 23; + public static final int UPDATE_ROWS_EVENT_V1 = 24; + public static final int DELETE_ROWS_EVENT_V1 = 25; + + /** + * Something out of the ordinary happened on the master + */ + public static final int INCIDENT_EVENT = 26; + + /** + * Heartbeat event to be send by master at its idle time to ensure master's online status to slave + */ + public static final int HEARTBEAT_LOG_EVENT = 27; + + /** + * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in + * case there is code for handling it, but which can be ignored if it is not recognized. + */ + public static final int IGNORABLE_LOG_EVENT = 28; + public static final int ROWS_QUERY_LOG_EVENT = 29; + + /** Version 2 of the Row events */ + public static final int WRITE_ROWS_EVENT = 30; + public static final int UPDATE_ROWS_EVENT = 31; + public static final int DELETE_ROWS_EVENT = 32; + + public static final int GTID_LOG_EVENT = 33; + public static final int ANONYMOUS_GTID_LOG_EVENT = 34; + + public static final int PREVIOUS_GTIDS_LOG_EVENT = 35; + + // mariaDb 5.5.34 + /* New MySQL/Sun events are to be added right above this comment */ + public static final int MYSQL_EVENTS_END = 36; + + public static final int MARIA_EVENTS_BEGIN = 160; + /* New Maria event numbers start from here */ + public static final int ANNOTATE_ROWS_EVENT = 160; + /* + Binlog checkpoint event. Used for XA crash recovery on the master, not used + in replication. + A binlog checkpoint event specifies a binlog file such that XA crash + recovery can start from that file - and it is guaranteed to find all XIDs + that are prepared in storage engines but not yet committed. + */ + public static final int BINLOG_CHECKPOINT_EVENT = 161; + /* + Gtid event. For global transaction ID, used to start a new event group, + instead of the old BEGIN query event, and also to mark stand-alone + events. + */ + public static final int GTID_EVENT = 162; + /* + Gtid list event. Logged at the start of every binlog, to record the + current replication state. This consists of the last GTID seen for + each replication domain. + */ + public static final int GTID_LIST_EVENT = 163; + + /** end marker */ + public static final int ENUM_END_EVENT = 164; + + /** + 1 byte length, 1 byte format + Length is total length in bytes, including 2 byte header + Length values 0 and 1 are currently invalid and reserved. + */ + public static final int EXTRA_ROW_INFO_LEN_OFFSET = 0; + public static final int EXTRA_ROW_INFO_FORMAT_OFFSET = 1; + public static final int EXTRA_ROW_INFO_HDR_BYTES = 2; + public static final int EXTRA_ROW_INFO_MAX_PAYLOAD = (255 - EXTRA_ROW_INFO_HDR_BYTES); + + // Events are without checksum though its generator + public static final int BINLOG_CHECKSUM_ALG_OFF = 0; + // is checksum-capable New Master (NM). + // CRC32 of zlib algorithm. + public static final int BINLOG_CHECKSUM_ALG_CRC32 = 1; + // the cut line: valid alg range is [1, 0x7f]. + public static final int BINLOG_CHECKSUM_ALG_ENUM_END = 2; + // special value to tag undetermined yet checksum + public static final int BINLOG_CHECKSUM_ALG_UNDEF = 255; + // or events from checksum-unaware servers + + public static final int CHECKSUM_CRC32_SIGNATURE_LEN = 4; + public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1; + /** + * defined statically while there is just one alg implemented + */ + public static final int BINLOG_CHECKSUM_LEN = CHECKSUM_CRC32_SIGNATURE_LEN; + + /* MySQL or old MariaDB slave with no announced capability. */ + public static final int MARIA_SLAVE_CAPABILITY_UNKNOWN = 0; + + /* MariaDB >= 5.3, which understands ANNOTATE_ROWS_EVENT. */ + public static final int MARIA_SLAVE_CAPABILITY_ANNOTATE = 1; + /* + * MariaDB >= 5.5. This version has the capability to tolerate events + * omitted from the binlog stream without breaking replication (MySQL slaves + * fail because they mis-compute the offsets into the master's binlog). + */ + public static final int MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES = 2; + /* MariaDB >= 10.0, which knows about binlog_checkpoint_log_event. */ + public static final int MARIA_SLAVE_CAPABILITY_BINLOG_CHECKPOINT = 3; + /* MariaDB >= 10.0.1, which knows about global transaction id events. */ + public static final int MARIA_SLAVE_CAPABILITY_GTID = 4; + + /* Our capability. */ + public static final int MARIA_SLAVE_CAPABILITY_MINE = MARIA_SLAVE_CAPABILITY_GTID; + + /** + For an event, 'e', carrying a type code, that a slave, + 's', does not recognize, 's' will check 'e' for + LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e' + is ignored. Otherwise, 's' acknowledges that it has + found an unknown event in the relay log. + */ + public static final int LOG_EVENT_IGNORABLE_F = 0x80; + + /** enum_field_types */ + public static final int MYSQL_TYPE_DECIMAL = 0; + public static final int MYSQL_TYPE_TINY = 1; + public static final int MYSQL_TYPE_SHORT = 2; + public static final int MYSQL_TYPE_LONG = 3; + public static final int MYSQL_TYPE_FLOAT = 4; + public static final int MYSQL_TYPE_DOUBLE = 5; + public static final int MYSQL_TYPE_NULL = 6; + public static final int MYSQL_TYPE_TIMESTAMP = 7; + public static final int MYSQL_TYPE_LONGLONG = 8; + public static final int MYSQL_TYPE_INT24 = 9; + public static final int MYSQL_TYPE_DATE = 10; + public static final int MYSQL_TYPE_TIME = 11; + public static final int MYSQL_TYPE_DATETIME = 12; + public static final int MYSQL_TYPE_YEAR = 13; + public static final int MYSQL_TYPE_NEWDATE = 14; + public static final int MYSQL_TYPE_VARCHAR = 15; + public static final int MYSQL_TYPE_BIT = 16; + public static final int MYSQL_TYPE_TIMESTAMP2 = 17; + public static final int MYSQL_TYPE_DATETIME2 = 18; + public static final int MYSQL_TYPE_TIME2 = 19; + public static final int MYSQL_TYPE_NEWDECIMAL = 246; + public static final int MYSQL_TYPE_ENUM = 247; + public static final int MYSQL_TYPE_SET = 248; + public static final int MYSQL_TYPE_TINY_BLOB = 249; + public static final int MYSQL_TYPE_MEDIUM_BLOB = 250; + public static final int MYSQL_TYPE_LONG_BLOB = 251; + public static final int MYSQL_TYPE_BLOB = 252; + public static final int MYSQL_TYPE_VAR_STRING = 253; + public static final int MYSQL_TYPE_STRING = 254; + public static final int MYSQL_TYPE_GEOMETRY = 255; + + public static String getTypeName(final int type) + { + switch (type) + { + case START_EVENT_V3: + return "Start_v3"; + case STOP_EVENT: + return "Stop"; + case QUERY_EVENT: + return "Query"; + case ROTATE_EVENT: + return "Rotate"; + case INTVAR_EVENT: + return "Intvar"; + case LOAD_EVENT: + return "Load"; + case NEW_LOAD_EVENT: + return "New_load"; + case SLAVE_EVENT: + return "Slave"; + case CREATE_FILE_EVENT: + return "Create_file"; + case APPEND_BLOCK_EVENT: + return "Append_block"; + case DELETE_FILE_EVENT: + return "Delete_file"; + case EXEC_LOAD_EVENT: + return "Exec_load"; + case RAND_EVENT: + return "RAND"; + case XID_EVENT: + return "Xid"; + case USER_VAR_EVENT: + return "User var"; + case FORMAT_DESCRIPTION_EVENT: + return "Format_desc"; + case TABLE_MAP_EVENT: + return "Table_map"; + case PRE_GA_WRITE_ROWS_EVENT: + return "Write_rows_event_old"; + case PRE_GA_UPDATE_ROWS_EVENT: + return "Update_rows_event_old"; + case PRE_GA_DELETE_ROWS_EVENT: + return "Delete_rows_event_old"; + case WRITE_ROWS_EVENT_V1: + return "Write_rows_v1"; + case UPDATE_ROWS_EVENT_V1: + return "Update_rows_v1"; + case DELETE_ROWS_EVENT_V1: + return "Delete_rows_v1"; + case BEGIN_LOAD_QUERY_EVENT: + return "Begin_load_query"; + case EXECUTE_LOAD_QUERY_EVENT: + return "Execute_load_query"; + case INCIDENT_EVENT: + return "Incident"; + case HEARTBEAT_LOG_EVENT: + return "Heartbeat"; + case IGNORABLE_LOG_EVENT: + return "Ignorable"; + case ROWS_QUERY_LOG_EVENT: + return "Rows_query"; + case WRITE_ROWS_EVENT: + return "Write_rows"; + case UPDATE_ROWS_EVENT: + return "Update_rows"; + case DELETE_ROWS_EVENT: + return "Delete_rows"; + case GTID_LOG_EVENT: + return "Gtid"; + case ANONYMOUS_GTID_LOG_EVENT: + return "Anonymous_Gtid"; + case PREVIOUS_GTIDS_LOG_EVENT: + return "Previous_gtids"; + default: + return "Unknown"; /* impossible */ + } + } + + protected static final Log logger = LogFactory.getLog(LogEvent.class); + + protected final LogHeader header; + + protected LogEvent(LogHeader header) + { + this.header = header; + } + + /** + * Return event header. + */ + public final LogHeader getHeader() + { + return header; + } + + /** + * The total size of this event, in bytes. In other words, this is the sum + * of the sizes of Common-Header, Post-Header, and Body. + */ + public final int getEventLen() + { + return header.getEventLen(); + } + + /** + * Server ID of the server that created the event. + */ + public final long getServerId() + { + return header.getServerId(); + } + + /** + * The position of the next event in the master binary log, in bytes from + * the beginning of the file. In a binlog that is not a relay log, this is + * just the position of the next event, in bytes from the beginning of the + * file. In a relay log, this is the position of the next event in the + * master's binlog. + */ + public final long getLogPos() + { + return header.getLogPos(); + } + + /** + * The time when the query started, in seconds since 1970. + */ + public final long getWhen() + { + return header.getWhen(); + } + + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java new file mode 100644 index 00000000..aca10a33 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java @@ -0,0 +1,93 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Arrays; + +/** + * Declaration a binary-log fetcher. It extends from LogBuffer. + * + * 
+ * LogFetcher fetcher = new SomeLogFetcher();
+ * ...
+ * 
+ * while (fetcher.fetch())
+ * {
+ *     LogEvent event;
+ *     do
+ *     {
+ *         event = decoder.decode(fetcher, context);
+ * 
+ *         // process log event.
+ *     }
+ *     while (event != null);
+ * }
+ * // no more binlog.
+ * fetcher.close();
+ *
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public abstract class LogFetcher extends LogBuffer implements Closeable +{ + /** Default initial capacity. */ + public static final int DEFAULT_INITIAL_CAPACITY = 8192; + + /** Default growth factor. */ + public static final float DEFAULT_GROWTH_FACTOR = 2.0f; + + /** Binlog file header size */ + public static final int BIN_LOG_HEADER_SIZE = 4; + + protected final float factor; + + public LogFetcher() + { + this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR); + } + + public LogFetcher(final int initialCapacity) + { + this(initialCapacity, DEFAULT_GROWTH_FACTOR); + } + + public LogFetcher(final int initialCapacity, final float growthFactor) + { + this.buffer = new byte[initialCapacity]; + this.factor = growthFactor; + } + + /** + * Increases the capacity of this LogFetcher instance, if + * necessary, to ensure that it can hold at least the number of elements + * specified by the minimum capacity argument. + * + * @param minCapacity the desired minimum capacity + */ + protected final void ensureCapacity(final int minCapacity) + { + final int oldCapacity = buffer.length; + + if (minCapacity > oldCapacity) + { + int newCapacity = (int) (oldCapacity * factor); + if (newCapacity < minCapacity) + newCapacity = minCapacity; + + buffer = Arrays.copyOf(buffer, newCapacity); + } + } + + /** + * Fetches the next frame of binary-log, and fill it in buffer. + */ + public abstract boolean fetch() throws IOException; + + /** + * {@inheritDoc} + * + * @see java.io.Closeable#close() + */ + public abstract void close() throws IOException; +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java new file mode 100644 index 00000000..1661e22a --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java @@ -0,0 +1,127 @@ +package com.taobao.tddl.dbsync.binlog; + +/** + * Implements binlog position. + * + * @author Changyuan.lh + * @version 1.0 + */ +public class LogPosition implements Cloneable, Comparable +{ + /* binlog file's name */ + protected String fileName; + + /* position in file */ + protected long position; + + /** + * Binlog position init. + * + * @param fileName file name for binlog files: mysql-bin.000001 + */ + public LogPosition(String fileName) + { + this.fileName = fileName; + this.position = 0L; + } + + /** + * Binlog position init. + * + * @param fileName file name for binlog files: mysql-bin.000001 + */ + public LogPosition(String fileName, final long position) + { + this.fileName = fileName; + this.position = position; + } + + /** + * Binlog position copy init. + */ + public LogPosition(LogPosition source) + { + this.fileName = source.fileName; + this.position = source.position; + } + + public final String getFileName() + { + return fileName; + } + + public final long getPosition() + { + return position; + } + + /* Clone binlog position without CloneNotSupportedException */ + public LogPosition clone() + { + try + { + return (LogPosition) super.clone(); + } + catch (CloneNotSupportedException e) + { + // Never happend + return null; + } + } + + /** + * Compares with the specified fileName and position. + */ + public final int compareTo(String fileName, final long position) + { + final int val = this.fileName.compareTo(fileName); + + if (val == 0) + { + return (int) (this.position - position); + } + return val; + } + + /** + * {@inheritDoc} + * + * @see java.lang.Comparable#compareTo(java.lang.Object) + */ + public int compareTo(LogPosition o) + { + final int val = fileName.compareTo(o.fileName); + + if (val == 0) + { + return (int) (position - o.position); + } + return val; + } + + /** + * {@inheritDoc} + * + * @see java.lang.Object#equals(java.lang.Object) + */ + public boolean equals(Object obj) + { + if (obj instanceof LogPosition) + { + LogPosition pos = ((LogPosition) obj); + return fileName.equals(pos.fileName) + && (this.position == pos.position); + } + return false; + } + + /** + * {@inheritDoc} + * + * @see java.lang.Object#toString() + */ + public String toString() + { + return fileName + ':' + position; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java new file mode 100644 index 00000000..4e459805 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java @@ -0,0 +1,53 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Append_block_log_event. + * + * @author Changyuan.lh + * @version 1.0 + */ +public class AppendBlockLogEvent extends LogEvent +{ + private final LogBuffer blockBuf; + private final int blockLen; + + private final long fileId; + + /* AB = "Append Block" */ + public static final int AB_FILE_ID_OFFSET = 0; + + public AppendBlockLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; + final int totalHeaderLen = commonHeaderLen + postHeaderLen; + + buffer.position(commonHeaderLen + AB_FILE_ID_OFFSET); + fileId = buffer.getUint32(); + + buffer.position(postHeaderLen); + blockLen = buffer.limit() - totalHeaderLen; + blockBuf = buffer.duplicate(blockLen); + } + + public final long getFileId() + { + return fileId; + } + + public final LogBuffer getBuffer() + { + return blockBuf; + } + + public final byte[] getData() + { + return blockBuf.getData(); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java new file mode 100644 index 00000000..df390862 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java @@ -0,0 +1,20 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * Event for the first block of file to be loaded, its only difference from + * Append_block event is that this event creates or truncates existing file + * before writing data. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent +{ + public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header, buffer, descriptionEvent); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java new file mode 100644 index 00000000..3d5accba --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java @@ -0,0 +1,73 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * Create_file_log_event. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class CreateFileLogEvent extends LoadLogEvent +{ + protected LogBuffer blockBuf; + protected int blockLen; + protected long fileId; + + protected boolean initedFromOld; + + /* CF = "Create File" */ + public static final int CF_FILE_ID_OFFSET = 0; + public static final int CF_DATA_OFFSET = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN; + + public CreateFileLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header, buffer, descriptionEvent); + + final int headerLen = descriptionEvent.commonHeaderLen; + final int loadHeaderLen = descriptionEvent.postHeaderLen[LOAD_EVENT - 1]; + final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1]; + + copyLogEvent(buffer, + ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen) + : (headerLen + loadHeaderLen + createFileHeaderLen)), + descriptionEvent); + + if (descriptionEvent.binlogVersion != 1) + { + fileId = buffer.getUint32(headerLen + loadHeaderLen + + CF_FILE_ID_OFFSET); + /* + Note that it's ok to use get_data_size() below, because it is computed + with values we have already read from this event (because we called + copy_log_event()); we are not using slave's format info to decode + master's format, we are really using master's format info. + Anyway, both formats should be identical (except the common_header_len) + as these Load events are not changed between 4.0 and 5.0 (as logging of + LOAD DATA INFILE does not use Load_log_event in 5.0). + */ + blockLen = buffer.limit() - buffer.position(); + blockBuf = buffer.duplicate(blockLen); + } + else + { + initedFromOld = true; + } + } + + public final long getFileId() + { + return fileId; + } + + public final LogBuffer getBuffer() + { + return blockBuf; + } + + public final byte[] getData() + { + return blockBuf.getData(); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java new file mode 100644 index 00000000..0fe3bae8 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java @@ -0,0 +1,33 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Delete_file_log_event. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class DeleteFileLogEvent extends LogEvent +{ + private final long fileId; + + /* DF = "Delete File" */ + public static final int DF_FILE_ID_OFFSET = 0; + + public DeleteFileLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET); + fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET + } + + public final long getFileId() + { + return fileId; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java new file mode 100644 index 00000000..01983d27 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java @@ -0,0 +1,19 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * Log row deletions. The event contain several delete rows for a table. Note + * that each event contains only rows for one table. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class DeleteRowsLogEvent extends RowsLogEvent +{ + public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header, buffer, descriptionEvent); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java new file mode 100644 index 00000000..664b51a6 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java @@ -0,0 +1,33 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Execute_load_log_event. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class ExecuteLoadLogEvent extends LogEvent +{ + private final long fileId; + + /* EL = "Execute Load" */ + public static final int EL_FILE_ID_OFFSET = 0; + + public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET); + fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET + } + + public final long getFileId() + { + return fileId; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java new file mode 100644 index 00000000..6a9e0f7e --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java @@ -0,0 +1,101 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.io.IOException; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * Event responsible for LOAD DATA execution, it similar to Query_log_event but + * before executing the query it substitutes original filename in LOAD DATA + * query with name of temporary file. + * + *
    + *
  • 4 bytes. The ID of the file to load.
    • + *
  • 4 bytes. The start position within the statement for filename + * substitution.
    • + *
  • 4 bytes. The end position within the statement for filename substitution. + *
    • + *
  • 1 byte. How to handle duplicates: LOAD_DUP_ERROR = 0, LOAD_DUP_IGNORE = + * 1, LOAD_DUP_REPLACE = 2
    • + *
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class ExecuteLoadQueryLogEvent extends QueryLogEvent +{ + /** file_id of temporary file */ + private long fileId; + + /** pointer to the part of the query that should be substituted */ + private int fnPosStart; + + /** pointer to the end of this part of query */ + private int fnPosEnd; + + /* + * Elements of this enum describe how LOAD DATA handles duplicates. + */ + public static final int LOAD_DUP_ERROR = 0; + public static final int LOAD_DUP_IGNORE = LOAD_DUP_ERROR + 1; + public static final int LOAD_DUP_REPLACE = LOAD_DUP_IGNORE + 1; + + /** + * We have to store type of duplicate handling explicitly, because for LOAD + * DATA it also depends on LOCAL option. And this part of query will be + * rewritten during replication so this information may be lost... + */ + private int dupHandling; + + /* ELQ = "Execute Load Query" */ + public static final int ELQ_FILE_ID_OFFSET = QUERY_HEADER_LEN; + public static final int ELQ_FN_POS_START_OFFSET = ELQ_FILE_ID_OFFSET + 4; + public static final int ELQ_FN_POS_END_OFFSET = ELQ_FILE_ID_OFFSET + 8; + public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12; + + public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) throws IOException + { + super(header, buffer, descriptionEvent); + + buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET); + + fileId = buffer.getUint32(); // ELQ_FILE_ID_OFFSET + fnPosStart = (int) buffer.getUint32(); // ELQ_FN_POS_START_OFFSET + fnPosEnd = (int) buffer.getUint32(); // ELQ_FN_POS_END_OFFSET + dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET + + final int len = query.length(); + if (fnPosStart > len || fnPosEnd > len + || dupHandling > LOAD_DUP_REPLACE) + { + throw new IOException(String.format( + "Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, " + + "fn_pos_end=%d, dup_handling=%d", fnPosStart, + fnPosEnd, dupHandling)); + } + } + + public final int getFilenamePosStart() + { + return fnPosStart; + } + + public final int getFilenamePosEnd() + { + return fnPosEnd; + } + + public final String getFilename() + { + if (query != null) + return query.substring(fnPosStart, fnPosEnd).trim(); + + return null; + } + + public final long getFileId() + { + return fileId; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java new file mode 100644 index 00000000..b2fe9352 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java @@ -0,0 +1,305 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.io.IOException; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the + * ordinary events). + * + * @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event + * @author Changyuan.lh + * @version 1.0 + */ +public final class FormatDescriptionLogEvent extends StartLogEventV3 { + + /** + * The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not + * exist in binlogs, it does not have a format). + */ + public static final int LOG_EVENT_TYPES = (ENUM_END_EVENT - 1); + + public static final int ST_COMMON_HEADER_LEN_OFFSET = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4); + + public static final int OLD_HEADER_LEN = 13; + public static final int LOG_EVENT_HEADER_LEN = 19; + public static final int LOG_EVENT_MINIMAL_HEADER_LEN = 19; + + /* event-specific post-header sizes */ + public static final int STOP_HEADER_LEN = 0; + public static final int LOAD_HEADER_LEN = (4 + 4 + 4 + 1 + 1 + 4); + public static final int SLAVE_HEADER_LEN = 0; + public static final int START_V3_HEADER_LEN = (2 + ST_SERVER_VER_LEN + 4); + public static final int ROTATE_HEADER_LEN = 8; // this + // is + // FROZEN + // (the + // Rotate + // post-header + // is + // frozen) + public static final int INTVAR_HEADER_LEN = 0; + public static final int CREATE_FILE_HEADER_LEN = 4; + public static final int APPEND_BLOCK_HEADER_LEN = 4; + public static final int EXEC_LOAD_HEADER_LEN = 4; + public static final int DELETE_FILE_HEADER_LEN = 4; + public static final int NEW_LOAD_HEADER_LEN = LOAD_HEADER_LEN; + public static final int RAND_HEADER_LEN = 0; + public static final int USER_VAR_HEADER_LEN = 0; + public static final int FORMAT_DESCRIPTION_HEADER_LEN = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES); + public static final int XID_HEADER_LEN = 0; + public static final int BEGIN_LOAD_QUERY_HEADER_LEN = APPEND_BLOCK_HEADER_LEN; + public static final int ROWS_HEADER_LEN_V1 = 8; + public static final int TABLE_MAP_HEADER_LEN = 8; + public static final int EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1); + public static final int EXECUTE_LOAD_QUERY_HEADER_LEN = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN); + public static final int INCIDENT_HEADER_LEN = 2; + public static final int HEARTBEAT_HEADER_LEN = 0; + public static final int IGNORABLE_HEADER_LEN = 0; + public static final int ROWS_HEADER_LEN_V2 = 10; + public static final int ANNOTATE_ROWS_HEADER_LEN = 0; + public static final int BINLOG_CHECKPOINT_HEADER_LEN = 4; + public static final int GTID_HEADER_LEN = 19; + public static final int GTID_LIST_HEADER_LEN = 4; + + public static final int POST_HEADER_LENGTH = 11; + + public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1; + public static final int[] checksumVersionSplit = { 5, 6, 1 }; + public static final long checksumVersionProduct = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1]) + * 256 + checksumVersionSplit[2]; + /** + * The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to + * LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size + * LOG_EVENT_MINIMAL_HEADER_LEN). + */ + protected final int commonHeaderLen; + protected int numberOfEventTypes; + + /** The list of post-headers' lengthes */ + protected final short[] postHeaderLen; + protected int[] serverVersionSplit = new int[3]; + + public FormatDescriptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent) + throws IOException{ + /* Start_log_event_v3 */ + super(header, buffer, descriptionEvent); + + buffer.position(LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET); + commonHeaderLen = buffer.getUint8(); + if (commonHeaderLen < OLD_HEADER_LEN) /* sanity check */ + { + throw new IOException("Format Description event header length is too short"); + } + + numberOfEventTypes = buffer.limit() - (LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET + 1); + + // buffer.position(LOG_EVENT_MINIMAL_HEADER_LEN + // + ST_COMMON_HEADER_LEN_OFFSET + 1); + postHeaderLen = new short[numberOfEventTypes]; + for (int i = 0; i < numberOfEventTypes; i++) { + postHeaderLen[i] = (short) buffer.getUint8(); + } + + calcServerVersionSplit(); + long calc = getVersionProduct(); + if (calc >= checksumVersionProduct) { + /* the last bytes are the checksum alg desc and value (or value's room) */ + numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN; + } + + if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= " + + numberOfEventTypes); + } + + /** MySQL 5.0 format descriptions. */ + public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_5_x = new FormatDescriptionLogEvent(4); + + /** MySQL 4.0.x (x>=2) format descriptions. */ + public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_4_0_x = new FormatDescriptionLogEvent(3); + + /** MySQL 3.23 format descriptions. */ + public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_3_23 = new FormatDescriptionLogEvent(1); + + public static FormatDescriptionLogEvent getFormatDescription(final int binlogVersion) throws IOException { + /* identify binlog format */ + switch (binlogVersion) { + case 4: /* MySQL 5.0 */ + return FORMAT_DESCRIPTION_EVENT_5_x; + case 3: + return FORMAT_DESCRIPTION_EVENT_4_0_x; + case 1: + return FORMAT_DESCRIPTION_EVENT_3_23; + default: + throw new IOException("Unknown binlog version: " + binlogVersion); + } + } + + public FormatDescriptionLogEvent(final int binlogVersion){ + this.binlogVersion = binlogVersion; + + postHeaderLen = new short[ENUM_END_EVENT]; + /* identify binlog format */ + switch (binlogVersion) { + case 4: /* MySQL 5.0 */ + serverVersion = SERVER_VERSION; + commonHeaderLen = LOG_EVENT_HEADER_LEN; + numberOfEventTypes = LOG_EVENT_TYPES; + + /* Note: all event types must explicitly fill in their lengths here. */ + postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN; + postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN; + postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN; + postHeaderLen[ROTATE_EVENT - 1] = ROTATE_HEADER_LEN; + postHeaderLen[INTVAR_EVENT - 1] = INTVAR_HEADER_LEN; + postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN; + postHeaderLen[SLAVE_EVENT - 1] = SLAVE_HEADER_LEN; + postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN; + postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN; + postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN; + postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN; + postHeaderLen[NEW_LOAD_EVENT - 1] = NEW_LOAD_HEADER_LEN; + postHeaderLen[RAND_EVENT - 1] = RAND_HEADER_LEN; + postHeaderLen[USER_VAR_EVENT - 1] = USER_VAR_HEADER_LEN; + postHeaderLen[FORMAT_DESCRIPTION_EVENT - 1] = FORMAT_DESCRIPTION_HEADER_LEN; + postHeaderLen[XID_EVENT - 1] = XID_HEADER_LEN; + postHeaderLen[BEGIN_LOAD_QUERY_EVENT - 1] = BEGIN_LOAD_QUERY_HEADER_LEN; + postHeaderLen[EXECUTE_LOAD_QUERY_EVENT - 1] = EXECUTE_LOAD_QUERY_HEADER_LEN; + postHeaderLen[TABLE_MAP_EVENT - 1] = TABLE_MAP_HEADER_LEN; + postHeaderLen[WRITE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1; + postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1; + postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1; + /* + * We here have the possibility to simulate a master of before we changed the table map id to be stored + * in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward + * compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know + * that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying + * test (rpl_row_4_bytes) too. + */ + postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0; + postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN; + postHeaderLen[ROWS_QUERY_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN; + postHeaderLen[WRITE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2; + postHeaderLen[UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2; + postHeaderLen[DELETE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2; + postHeaderLen[GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH; + postHeaderLen[ANONYMOUS_GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH; + postHeaderLen[PREVIOUS_GTIDS_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN; + // mariadb 10 + postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN; + postHeaderLen[BINLOG_CHECKPOINT_EVENT - 1] = BINLOG_CHECKPOINT_HEADER_LEN; + postHeaderLen[GTID_EVENT - 1] = GTID_HEADER_LEN; + postHeaderLen[GTID_LIST_EVENT - 1] = GTID_LIST_HEADER_LEN; + break; + + case 3: /* 4.0.x x>=2 */ + /* + * We build an artificial (i.e. not sent by the master) event, which describes what those old master + * versions send. + */ + serverVersion = "4.0"; + commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN; + + /* + * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist + * in older versions. We use the events known by version 3, even if version 1 had only a subset of them + * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave + * detect less corruptions). + */ + numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1; + + postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN; + postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_MINIMAL_LEN; + postHeaderLen[ROTATE_EVENT - 1] = ROTATE_HEADER_LEN; + postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN; + postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN; + postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN; + postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN; + postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN; + postHeaderLen[NEW_LOAD_EVENT - 1] = postHeaderLen[LOAD_EVENT - 1]; + break; + + case 1: /* 3.23 */ + /* + * We build an artificial (i.e. not sent by the master) event, which describes what those old master + * versions send. + */ + serverVersion = "3.23"; + commonHeaderLen = OLD_HEADER_LEN; + + /* + * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist + * in older versions. We use the events known by version 3, even if version 1 had only a subset of them + * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave + * detect less corruptions). + */ + numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1; + + postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN; + postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_MINIMAL_LEN; + postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN; + postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN; + postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN; + postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN; + postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN; + postHeaderLen[NEW_LOAD_EVENT - 1] = postHeaderLen[LOAD_EVENT - 1]; + break; + + default: + numberOfEventTypes = 0; + commonHeaderLen = 0; + } + } + + public void calcServerVersionSplit() { + doServerVersionSplit(serverVersion, serverVersionSplit); + } + + public long getVersionProduct() { + return versionProduct(serverVersionSplit); + } + + public boolean isVersionBeforeChecksum() { + return getVersionProduct() < checksumVersionProduct; + } + + public static void doServerVersionSplit(String serverVersion, int[] versionSplit) { + String[] split = serverVersion.split("\\."); + if (split.length < 3) { + versionSplit[0] = 0; + versionSplit[1] = 0; + versionSplit[2] = 0; + } else { + int j = 0; + for (int i = 0; i <= 2; i++) { + String str = split[i]; + for (j = 0; j < str.length(); j++) { + if (Character.isDigit(str.charAt(j)) == false) { + break; + } + } + if (j > 0) { + versionSplit[i] = Integer.valueOf(str.substring(0, j), 10); + } else { + versionSplit[0] = 0; + versionSplit[1] = 0; + versionSplit[2] = 0; + } + } + } + } + + public static long versionProduct(int[] versionSplit) { + return ((versionSplit[0] * 256 + versionSplit[1]) * 256 + versionSplit[2]); + } + + public final int getCommonHeaderLen() { + return commonHeaderLen; + } + + public final short[] getPostHeaderLen() { + return postHeaderLen; + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java new file mode 100644 index 00000000..d1684d35 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java @@ -0,0 +1,45 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * @author jianghang 2013-4-8 上午12:36:29 + * @version 1.0.3 + * @since mysql 5.6 / mariadb10 + */ +public class GtidLogEvent extends LogEvent { + + // / Length of the commit_flag in event encoding + public static final int ENCODED_FLAG_LENGTH = 1; + // / Length of SID in event encoding + public static final int ENCODED_SID_LENGTH = 16; + + private boolean commitFlag; + + public GtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + // final int postHeaderLen = descriptionEvent.postHeaderLen[header.type + // - 1]; + + buffer.position(commonHeaderLen); + commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH + + // ignore gtid info read + // sid.copy_from((uchar *)ptr_buffer); + // ptr_buffer+= ENCODED_SID_LENGTH; + // + // // SIDNO is only generated if needed, in get_sidno(). + // spec.gtid.sidno= -1; + // + // spec.gtid.gno= uint8korr(ptr_buffer); + // ptr_buffer+= ENCODED_GNO_LENGTH; + } + + public boolean isCommitFlag() { + return commitFlag; + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java new file mode 100644 index 00000000..f3948d7e --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java @@ -0,0 +1,49 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * 
+ * Replication event to ensure to slave that master is alive.
+ *   The event is originated by master's dump thread and sent straight to
+ *   slave without being logged. Slave itself does not store it in relay log
+ *   but rather uses a data for immediate checks and throws away the event.
+ * 
+ *   Two members of the class log_ident and Log_event::log_pos comprise 
+ *   @see the event_coordinates instance. The coordinates that a heartbeat
+ *   instance carries correspond to the last event master has sent from
+ *   its binlog.
+ *
+ * + * @author jianghang 2013-4-8 上午12:36:29 + * @version 1.0.3 + * @since mysql 5.6 + */ +public class HeartbeatLogEvent extends LogEvent { + + public static final int FN_REFLEN = 512; /* Max length of full path-name */ + private int identLen; + private String logIdent; + + public HeartbeatLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + identLen = buffer.limit() - commonHeaderLen; + if (identLen > FN_REFLEN - 1) { + identLen = FN_REFLEN - 1; + } + + logIdent = buffer.getFullString(commonHeaderLen, identLen, LogBuffer.ISO_8859_1); + } + + public int getIdentLen() { + return identLen; + } + + public String getLogIdent() { + return logIdent; + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java new file mode 100644 index 00000000..ccc49762 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java @@ -0,0 +1,35 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * 
+ *   Base class for ignorable log events. Events deriving from
+  this class can be safely ignored by slaves that cannot
+  recognize them. Newer slaves, will be able to read and
+  handle them. This has been designed to be an open-ended
+  architecture, so adding new derived events shall not harm
+  the old slaves that support ignorable log event mechanism
+  (they will just ignore unrecognized ignorable events).
+
+  @note The only thing that makes an event ignorable is that it has
+  the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
+  that ignorable event types derive from Ignorable_log_event; they may
+  just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
+  argument to the Log_event constructor.
+
+ * + * @author jianghang 2013-4-8 上午12:36:29 + * @version 1.0.3 + * @since mysql 5.6 + */ +public class IgnorableLogEvent extends LogEvent { + + public IgnorableLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header); + + // do nothing , just ignore log event + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java new file mode 100644 index 00000000..ecbd3226 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java @@ -0,0 +1,88 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Class representing an incident, an occurance out of the ordinary, that + * happened on the master. + * + * The event is used to inform the slave that something out of the ordinary + * happened on the master that might cause the database to be in an inconsistent + * state. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Incident event format
SymbolFormatDescription
INCIDENT2Incident number as an unsigned integer
MSGLEN1Message length as an unsigned integer
MESSAGEMSGLENThe message, if present. Not null terminated.
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class IncidentLogEvent extends LogEvent +{ + public static final int INCIDENT_NONE = 0; + + /** There are possibly lost events in the replication stream */ + public static final int INCIDENT_LOST_EVENTS = 1; + + /** Shall be last event of the enumeration */ + public static final int INCIDENT_COUNT = 2; + + private final int incident; + private final String message; + + public IncidentLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; + + buffer.position(commonHeaderLen); + final int incidentNumber = buffer.getUint16(); + if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE) + { + // If the incident is not recognized, this binlog event is + // invalid. If we set incident_number to INCIDENT_NONE, the + // invalidity will be detected by is_valid(). + incident = INCIDENT_NONE; + message = null; + return; + } + incident = incidentNumber; + + buffer.position(commonHeaderLen + postHeaderLen); + message = buffer.getString(); + } + + public final int getIncident() + { + return incident; + } + + public final String getMessage() + { + return message; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java new file mode 100644 index 00000000..9cce9b2c --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java @@ -0,0 +1,98 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * An Intvar_log_event will be created just before a Query_log_event, if the + * query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each + * Intvar_log_event holds the value of one of these variables. + * + * Binary Format + * + * The Post-Header for this event type is empty. The Body has two components: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Body for Intvar_log_event
NameFormatDescription
type1 byte enumerationOne byte identifying the type of variable stored. Currently, two + * identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.
value8 byte unsigned integerThe value of the variable.
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class IntvarLogEvent extends LogEvent +{ + /** + * Fixed data part: Empty + * + *

+ * Variable data part: + * + *

    + *
  • 1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT = + * 1 or INSERT_ID_EVENT = 2.
    • + *
  • 8 bytes. An unsigned integer indicating the value to be used for the + * LAST_INSERT_ID() invocation or AUTO_INCREMENT column.
    • + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + private final long value; + private final int type; + + /* Intvar event data */ + public static final int I_TYPE_OFFSET = 0; + public static final int I_VAL_OFFSET = 1; + + // enum Int_event_type + public static final int INVALID_INT_EVENT = 0; + public static final int LAST_INSERT_ID_EVENT = 1; + public static final int INSERT_ID_EVENT = 2; + + public IntvarLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + /* The Post-Header is empty. The Varible Data part begins immediately. */ + buffer.position(descriptionEvent.commonHeaderLen + + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1] + + I_TYPE_OFFSET); + type = buffer.getInt8(); // I_TYPE_OFFSET + value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET); + } + + public final int getType() + { + return type; + } + + public final long getValue() + { + return value; + } + + public final String getQuery() + { + return "SET INSERT_ID = " + value; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java new file mode 100644 index 00000000..adfeae52 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java @@ -0,0 +1,391 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * This log event corresponds to a "LOAD DATA INFILE" SQL query on the following + * form: + * + * 
+ *    (1)    USE db;
+ *    (2)    LOAD DATA [CONCURRENT] [LOCAL] INFILE 'file_name'
+ *    (3)    [REPLACE | IGNORE]
+ *    (4)    INTO TABLE 'table_name'
+ *    (5)    [FIELDS
+ *    (6)      [TERMINATED BY 'field_term']
+ *    (7)      [[OPTIONALLY] ENCLOSED BY 'enclosed']
+ *    (8)      [ESCAPED BY 'escaped']
+ *    (9)    ]
+ *   (10)    [LINES
+ *   (11)      [TERMINATED BY 'line_term']
+ *   (12)      [LINES STARTING BY 'line_start']
+ *   (13)    ]
+ *   (14)    [IGNORE skip_lines LINES]
+ *   (15)    (field_1, field_2, ..., field_n)
+ *
+ * + * Binary Format: The Post-Header consists of the following six components. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Post-Header for Load_log_event
NameFormatDescription
slave_proxy_id4 byte unsigned integerAn integer identifying the client thread that issued the query. The id is + * unique per server. (Note, however, that two threads on different servers may + * have the same slave_proxy_id.) This is used when a client thread creates a + * temporary table local to the client. The slave_proxy_id is used to + * distinguish temporary tables that belong to different clients.
exec_time4 byte unsigned integerThe time from when the query started to when it was logged in the binlog, + * in seconds.
skip_lines4 byte unsigned integerThe number on line (14) above, if present, or 0 if line (14) is left out. + *
table_name_len1 byte unsigned integerThe length of 'table_name' on line (4) above.
db_len1 byte unsigned integerThe length of 'db' on line (1) above.
num_fields4 byte unsigned integerThe number n of fields on line (15) above.
+ * + * The Body contains the following components. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Body of Load_log_event
NameFormatDescription
sql_exvariable lengthDescribes the part of the query on lines (3) and (5)–(13) above. + * More precisely, it stores the five strings (on lines) field_term (6), + * enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a + * bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and + * OPTIONALLY (7). + * + * The data is stored in one of two formats, called "old" and "new". The type + * field of Common-Header determines which of these two formats is used: type + * LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means + * that the new format is used. When MySQL writes a Load_log_event, it uses the + * new format if at least one of the five strings is two or more bytes long. + * Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is + * used. + * + * The new and old format differ in the way the five strings are stored. + * + *
    + *
  • In the new format, the strings are stored in the order field_term, + * enclosed, escaped, line_term, line_start. Each string consists of a length (1 + * byte), followed by a sequence of characters (0-255 bytes). Finally, a boolean + * combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4, + * IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates + * the presence of the corresponding keyword in the SQL query. + * + *
  • In the old format, we know that each string has length 0 or 1. Therefore, + * only the first byte of each string is stored. The order of the strings is the + * same as in the new format. These five bytes are followed by the same 1 byte + * bitfield as in the new format. Finally, a 1 byte bitfield called empty_flags + * is stored. The low 5 bits of empty_flags indicate which of the five strings + * have length 0. For each of the following flags that is set, the corresponding + * string has length 0; for the flags that are not set, the string has length 1: + * FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4, + * LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10. + *
+ * + * Thus, the size of the new format is 6 bytes + the sum of the sizes of the + * five strings. The size of the old format is always 7 bytes.
field_lensnum_fields 1 byte unsigned integersAn array of num_fields integers representing the length of each field in + * the query. (num_fields is from the Post-Header).
fieldsnum_fields null-terminated stringsAn array of num_fields null-terminated strings, each representing a field + * in the query. (The trailing zero is redundant, since the length are stored in + * the num_fields array.) The total length of all strings equals to the sum of + * all field_lens, plus num_fields bytes for all the trailing zeros.
table_namenull-terminated string of length table_len+1 bytesThe 'table_name' from the query, as a null-terminated string. (The + * trailing zero is actually redundant since the table_len is known from + * Post-Header.)
dbnull-terminated string of length db_len+1 bytesThe 'db' from the query, as a null-terminated string. (The trailing zero + * is actually redundant since the db_len is known from Post-Header.)
file_namevariable length string without trailing zero, extending to the end of the + * event (determined by the length field of the Common-Header)The 'file_name' from the query.
+ * + * This event type is understood by current versions, but only generated by + * MySQL 3.23 and earlier. + * + * @author Changyuan.lh + * @version 1.0 + */ +public class LoadLogEvent extends LogEvent +{ + private String table; + private String db; + private String fname; + private int skipLines; + private int numFields; + private String[] fields; + + /* sql_ex_info */ + private String fieldTerm; + private String lineTerm; + private String lineStart; + private String enclosed; + private String escaped; + private int optFlags; + private int emptyFlags; + + private long execTime; + + /* Load event post-header */ + public static final int L_THREAD_ID_OFFSET = 0; + public static final int L_EXEC_TIME_OFFSET = 4; + public static final int L_SKIP_LINES_OFFSET = 8; + public static final int L_TBL_LEN_OFFSET = 12; + public static final int L_DB_LEN_OFFSET = 13; + public static final int L_NUM_FIELDS_OFFSET = 14; + public static final int L_SQL_EX_OFFSET = 18; + public static final int L_DATA_OFFSET = FormatDescriptionLogEvent.LOAD_HEADER_LEN; + + /* + These are flags and structs to handle all the LOAD DATA INFILE options (LINES + TERMINATED etc). + DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD + DATA). + */ + public static final int DUMPFILE_FLAG = 0x1; + public static final int OPT_ENCLOSED_FLAG = 0x2; + public static final int REPLACE_FLAG = 0x4; + public static final int IGNORE_FLAG = 0x8; + + public static final int FIELD_TERM_EMPTY = 0x1; + public static final int ENCLOSED_EMPTY = 0x2; + public static final int LINE_TERM_EMPTY = 0x4; + public static final int LINE_START_EMPTY = 0x8; + public static final int ESCAPED_EMPTY = 0x10; + + public LoadLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN; + /* + * I (Guilhem) manually tested replication of LOAD DATA INFILE for + * 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works. + */ + copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen + + descriptionEvent.commonHeaderLen : loadHeaderLen + + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN), + descriptionEvent); + } + + /** + * @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event + */ + protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset, + FormatDescriptionLogEvent descriptionEvent) + { + /* this is the beginning of the post-header */ + buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET); + + execTime = buffer.getUint32(); // L_EXEC_TIME_OFFSET + skipLines = (int) buffer.getUint32(); // L_SKIP_LINES_OFFSET + final int tableNameLen = buffer.getUint8(); // L_TBL_LEN_OFFSET + final int dbLen = buffer.getUint8(); // L_DB_LEN_OFFSET + numFields = (int) buffer.getUint32(); // L_NUM_FIELDS_OFFSET + + buffer.position(bodyOffset); + /* + * Sql_ex.init() on success returns the pointer to the first byte after + * the sql_ex structure, which is the start of field lengths array. + */ + if (header.type != LOAD_EVENT /* use_new_format */) + { + /* + * The code below assumes that buf will not disappear from under our + * feet during the lifetime of the event. This assumption holds true + * in the slave thread if the log is in new format, but is not the + * case when we have old format because we will be reusing net + * buffer to read the actual file before we write out the + * Create_file event. + */ + fieldTerm = buffer.getString(); + enclosed = buffer.getString(); + lineTerm = buffer.getString(); + lineStart = buffer.getString(); + escaped = buffer.getString(); + optFlags = buffer.getInt8(); + emptyFlags = 0; + } + else + { + fieldTerm = buffer.getFixString(1); + enclosed = buffer.getFixString(1); + lineTerm = buffer.getFixString(1); + lineStart = buffer.getFixString(1); + escaped = buffer.getFixString(1); + optFlags = buffer.getUint8(); + emptyFlags = buffer.getUint8(); + + if ((emptyFlags & FIELD_TERM_EMPTY) != 0) + fieldTerm = null; + if ((emptyFlags & ENCLOSED_EMPTY) != 0) + enclosed = null; + if ((emptyFlags & LINE_TERM_EMPTY) != 0) + lineTerm = null; + if ((emptyFlags & LINE_START_EMPTY) != 0) + lineStart = null; + if ((emptyFlags & ESCAPED_EMPTY) != 0) + escaped = null; + } + + final int fieldLenPos = buffer.position(); + buffer.forward(numFields); + fields = new String[numFields]; + for (int i = 0; i < numFields; i++) + { + final int fieldLen = buffer.getUint8(fieldLenPos + i); + fields[i] = buffer.getFixString(fieldLen + 1); + } + + table = buffer.getFixString(tableNameLen + 1); + db = buffer.getFixString(dbLen + 1); + + // null termination is accomplished by the caller + final int from = buffer.position(); + final int end = from + buffer.limit(); + int found = from; + for (; (found < end) && buffer.getInt8(found) != '\0'; found++) + /* empty loop */; + fname = buffer.getString(found); + buffer.forward(1); // The + 1 is for \0 terminating fname + } + + public final String getTable() + { + return table; + } + + public final String getDb() + { + return db; + } + + public final String getFname() + { + return fname; + } + + public final int getSkipLines() + { + return skipLines; + } + + public final String[] getFields() + { + return fields; + } + + public final String getFieldTerm() + { + return fieldTerm; + } + + public final String getLineTerm() + { + return lineTerm; + } + + public final String getLineStart() + { + return lineStart; + } + + public final String getEnclosed() + { + return enclosed; + } + + public final String getEscaped() + { + return escaped; + } + + public final int getOptFlags() + { + return optFlags; + } + + public final int getEmptyFlags() + { + return emptyFlags; + } + + public final long getExecTime() + { + return execTime; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java new file mode 100644 index 00000000..c1b72b45 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java @@ -0,0 +1,310 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * The Common-Header, documented in the table @ref Table_common_header "below", + * always has the same form and length within one version of MySQL. Each event + * type specifies a format and length of the Post-Header. The length of the + * Common-Header is the same for all events of the same type. The Body may be of + * different format and length even for different events of the same type. The + * binary formats of Post-Header and Body are documented separately in each + * subclass. The binary format of Common-Header is as follows. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Common-Header
NameFormatDescription
timestamp4 byte unsigned integerThe time when the query started, in seconds since 1970.
type1 byte enumerationSee enum #Log_event_type.
server_id4 byte unsigned integerServer ID of the server that created the event.
total_size4 byte unsigned integerThe total size of this event, in bytes. In other words, this is the sum + * of the sizes of Common-Header, Post-Header, and Body.
master_position4 byte unsigned integerThe position of the next event in the master binary log, in bytes from + * the beginning of the file. In a binlog that is not a relay log, this is just + * the position of the next event, in bytes from the beginning of the file. In a + * relay log, this is the position of the next event in the master's binlog.
flags2 byte bitfieldSee Log_event::flags.
+ * + * Summing up the numbers above, we see that the total size of the common header + * is 19 bytes. + * + * @see mysql-5.1.60/sql/log_event.cc + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class LogHeader +{ + protected final int type; + + /** + * The offset in the log where this event originally appeared (it is + * preserved in relay logs, making SHOW SLAVE STATUS able to print + * coordinates of the event in the master's binlog). Note: when a + * transaction is written by the master to its binlog (wrapped in + * BEGIN/COMMIT) the log_pos of all the queries it contains is the one of + * the BEGIN (this way, when one does SHOW SLAVE STATUS it sees the offset + * of the BEGIN, which is logical as rollback may occur), except the COMMIT + * query which has its real offset. + */ + protected long logPos; + + /** + * Timestamp on the master(for debugging and replication of + * NOW()/TIMESTAMP). It is important for queries and LOAD DATA INFILE. This + * is set at the event's creation time, except for Query and Load (et al.) + * events where this is set at the query's execution time, which guarantees + * good replication (otherwise, we could have a query and its event with + * different timestamps). + */ + protected long when; + + /** Number of bytes written by write() function */ + protected int eventLen; + + /** + * The master's server id (is preserved in the relay log; used to prevent + * from infinite loops in circular replication). + */ + protected long serverId; + + /** + * Some 16 flags. See the definitions above for LOG_EVENT_TIME_F, + * LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and + * LOG_EVENT_SUPPRESS_USE_F for notes. + */ + protected int flags; + + /** + * The value is set by caller of FD constructor and + * Log_event::write_header() for the rest. + * In the FD case it's propagated into the last byte + * of post_header_len[] at FD::write(). + * On the slave side the value is assigned from post_header_len[last] + * of the last seen FD event. + */ + protected int checksumAlg; + /** + Placeholder for event checksum while writing to binlog. + */ + protected long crc; // ha_checksum + + /* for Start_event_v3 */ + public LogHeader(final int type) + { + this.type = type; + } + + public LogHeader(LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + when = buffer.getUint32(); + type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET; + serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET; + eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET; + + if (descriptionEvent.binlogVersion == 1) + { + logPos = 0; + flags = 0; + return; + } + + /* 4.0 or newer */ + logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET + /* + If the log is 4.0 (so here it can only be a 4.0 relay log read by + the SQL thread or a 4.0 master binlog read by the I/O thread), + log_pos is the beginning of the event: we transform it into the end + of the event, which is more useful. + But how do you know that the log is 4.0: you know it if + description_event is version 3 *and* you are not reading a + Format_desc (remember that mysqlbinlog starts by assuming that 5.0 + logs are in 4.0 format, until it finds a Format_desc). + */ + if (descriptionEvent.binlogVersion == 3 + && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0) + { + /* + If log_pos=0, don't change it. log_pos==0 is a marker to mean + "don't change rli->group_master_log_pos" (see + inc_group_relay_log_pos()). As it is unreal log_pos, adding the + event len's is nonsense. For example, a fake Rotate event should + not have its log_pos (which is 0) changed or it will modify + Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense + value of (a non-zero offset which does not exist in the master's + binlog, so which will cause problems if the user uses this value + in CHANGE MASTER). + */ + logPos += eventLen; /* purecov: inspected */ + } + + flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET + if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT) + || (type == LogEvent.ROTATE_EVENT)) + { + /* + These events always have a header which stops here (i.e. their + header is FROZEN). + */ + /* + Initialization to zero of all other Log_event members as they're + not specified. Currently there are no such members; in the future + there will be an event UID (but Format_description and Rotate + don't need this UID, as they are not propagated through + --log-slave-updates (remember the UID is used to not play a query + twice when you have two masters which are slaves of a 3rd master). + Then we are done. + */ + + if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) { + int commonHeaderLen = buffer.getUint8(FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN + + FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET); + buffer.position(commonHeaderLen + FormatDescriptionLogEvent.ST_SERVER_VER_OFFSET); + String serverVersion = buffer.getFixString(FormatDescriptionLogEvent.ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET + int versionSplit[] = new int[] { 0, 0, 0 }; + FormatDescriptionLogEvent.doServerVersionSplit(serverVersion, versionSplit); + checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF; + if (FormatDescriptionLogEvent.versionProduct(versionSplit) >= FormatDescriptionLogEvent.checksumVersionProduct) { + buffer.position(eventLen - LogEvent.BINLOG_CHECKSUM_LEN - LogEvent.BINLOG_CHECKSUM_ALG_DESC_LEN); + checksumAlg = buffer.getUint8(); + } + + processCheckSum(buffer); + } + return; + } + + /* + CRC verification by SQL and Show-Binlog-Events master side. + The caller has to provide @description_event->checksum_alg to + be the last seen FD's (A) descriptor. + If event is FD the descriptor is in it. + Notice, FD of the binlog can be only in one instance and therefore + Show-Binlog-Events executing master side thread needs just to know + the only FD's (A) value - whereas RL can contain more. + In the RL case, the alg is kept in FD_e (@description_event) which is reset + to the newer read-out event after its execution with possibly new alg descriptor. + Therefore in a typical sequence of RL: + {FD_s^0, FD_m, E_m^1} E_m^1 + will be verified with (A) of FD_m. + + See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs + lines (log.h). + + Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF. + */ + checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg + processCheckSum(buffer); + /* otherwise, go on with reading the header from buf (nothing now) */ + } + + /** + * The different types of log events. + */ + public final int getType() + { + return type; + } + + /** + * The position of the next event in the master binary log, in bytes from + * the beginning of the file. In a binlog that is not a relay log, this is + * just the position of the next event, in bytes from the beginning of the + * file. In a relay log, this is the position of the next event in the + * master's binlog. + */ + public final long getLogPos() + { + return logPos; + } + + /** + * The total size of this event, in bytes. In other words, this is the sum + * of the sizes of Common-Header, Post-Header, and Body. + */ + public final int getEventLen() + { + return eventLen; + } + + /** + * The time when the query started, in seconds since 1970. + */ + public final long getWhen() + { + return when; + } + + /** + * Server ID of the server that created the event. + */ + public final long getServerId() + { + return serverId; + } + + /** + * Some 16 flags. See the definitions above for LOG_EVENT_TIME_F, + * LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and + * LOG_EVENT_SUPPRESS_USE_F for notes. + */ + public final int getFlags() + { + return flags; + } + + + public long getCrc() { + return crc; + } + + + public int getChecksumAlg() { + return checksumAlg; + } + + private void processCheckSum(LogBuffer buffer) { + if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && + checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){ + crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN); + } + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java new file mode 100644 index 00000000..246096f5 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java @@ -0,0 +1,19 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * + * @author jianghang 2013-4-8 上午12:36:29 + * @version 1.0.3 + * @since mysql 5.6 + */ +public class PreviousGtidsLogEvent extends LogEvent{ + + public PreviousGtidsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header); + // do nothing , just for mysql gtid search function + } +} + diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java new file mode 100644 index 00000000..d9306e51 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java @@ -0,0 +1,920 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.io.IOException; +import java.nio.charset.Charset; + +import com.taobao.tddl.dbsync.binlog.CharsetConversion; +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * A Query_log_event is created for each query that modifies the database, + * unless the query is logged row-based. + * + * The Post-Header has five components: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Post-Header for Query_log_event
NameFormatDescription
slave_proxy_id4 byte unsigned integerAn integer identifying the client thread that issued the query. The id is + * unique per server. (Note, however, that two threads on different servers may + * have the same slave_proxy_id.) This is used when a client thread creates a + * temporary table local to the client. The slave_proxy_id is used to + * distinguish temporary tables that belong to different clients.
exec_time4 byte unsigned integerThe time from when the query started to when it was logged in the binlog, + * in seconds.
db_len1 byte integerThe length of the name of the currently selected database.
error_code2 byte unsigned integerError code generated by the master. If the master fails, the slave will + * fail with the same error code, except for the error codes ER_DB_CREATE_EXISTS + * == 1007 and ER_DB_DROP_EXISTS == 1008.
status_vars_len2 byte unsigned integerThe length of the status_vars block of the Body, in bytes. See + * query_log_event_status_vars "below".
+ * + * The Body has the following components: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Body for Query_log_event
NameFormatDescription
query_log_event_status_vars status_varsstatus_vars_len bytesZero or more status variables. Each status variable consists of one byte + * identifying the variable stored, followed by the value of the variable. The + * possible variables are listed separately in the table + * Table_query_log_event_status_vars "below". MySQL always writes events in the + * order defined below; however, it is capable of reading them in any order.
dbdb_len+1The currently selected database, as a null-terminated string. + * + * (The trailing zero is redundant since the length is already known; it is + * db_len from Post-Header.)
queryvariable length string without trailing zero, extending to the end of the + * event (determined by the length field of the Common-Header)The SQL query.
+ * + * The following table lists the status variables that may appear in the + * status_vars field. Table_query_log_event_status_vars + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Status variables for Query_log_event
Status variable1 byte identifierFormatDescription
flags2Q_FLAGS2_CODE == 04 byte bitfieldThe flags in thd->options, binary AND-ed with OPTIONS_WRITTEN_TO_BIN_LOG. + * The thd->options bitfield contains options for "SELECT". OPTIONS_WRITTEN + * identifies those options that need to be written to the binlog (not all do). + * Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL | + * OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS | + * OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. + * + * These flags correspond to the SQL variables SQL_AUTO_IS_NULL, + * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and AUTOCOMMIT, documented in the + * "SET Syntax" section of the MySQL Manual. + * + * This field is always written to the binlog in version >= 5.0, and never + * written in version < 5.0.
sql_modeQ_SQL_MODE_CODE == 18 byte bitfieldThe sql_mode variable. See the section "SQL Modes" in the MySQL manual, + * and see mysql_priv.h for a list of the possible flags. Currently + * (2007-10-04), the following flags are available: + * + * 
+ *     MODE_REAL_AS_FLOAT==0x1
+ *     MODE_PIPES_AS_CONCAT==0x2
+ *     MODE_ANSI_QUOTES==0x4
+ *     MODE_IGNORE_SPACE==0x8
+ *     MODE_NOT_USED==0x10
+ *     MODE_ONLY_FULL_GROUP_BY==0x20
+ *     MODE_NO_UNSIGNED_SUBTRACTION==0x40
+ *     MODE_NO_DIR_IN_CREATE==0x80
+ *     MODE_POSTGRESQL==0x100
+ *     MODE_ORACLE==0x200
+ *     MODE_MSSQL==0x400
+ *     MODE_DB2==0x800
+ *     MODE_MAXDB==0x1000
+ *     MODE_NO_KEY_OPTIONS==0x2000
+ *     MODE_NO_TABLE_OPTIONS==0x4000
+ *     MODE_NO_FIELD_OPTIONS==0x8000
+ *     MODE_MYSQL323==0x10000
+ *     MODE_MYSQL323==0x20000
+ *     MODE_MYSQL40==0x40000
+ *     MODE_ANSI==0x80000
+ *     MODE_NO_AUTO_VALUE_ON_ZERO==0x100000
+ *     MODE_NO_BACKSLASH_ESCAPES==0x200000
+ *     MODE_STRICT_TRANS_TABLES==0x400000
+ *     MODE_STRICT_ALL_TABLES==0x800000
+ *     MODE_NO_ZERO_IN_DATE==0x1000000
+ *     MODE_NO_ZERO_DATE==0x2000000
+ *     MODE_INVALID_DATES==0x4000000
+ *     MODE_ERROR_FOR_DIVISION_BY_ZERO==0x8000000
+ *     MODE_TRADITIONAL==0x10000000
+ *     MODE_NO_AUTO_CREATE_USER==0x20000000
+ *     MODE_HIGH_NOT_PRECEDENCE==0x40000000
+ *     MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000
+ *
+ * + * All these flags are replicated from the server. However, all flags except + * MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always preserves + * its old value of MODE_NO_DIR_IN_CREATE. For a rationale, see comment in + * Query_log_event::do_apply_event in log_event.cc. + * + * This field is always written to the binlog.
catalogQ_CATALOG_NZ_CODE == 6Variable-length string: the length in bytes (1 byte) followed by the + * characters (at most 255 bytes)Stores the client's current catalog. Every database belongs to a catalog, + * the same way that every table belongs to a database. Currently, there is only + * one catalog, "std". + * + * This field is written if the length of the catalog is > 0; otherwise it is + * not written.
auto_incrementQ_AUTO_INCREMENT == 3two 2 byte unsigned integers, totally 2+2=4 bytesThe two variables auto_increment_increment and auto_increment_offset, in + * that order. For more information, see "System variables" in the MySQL manual. + * + * This field is written if auto_increment > 1. Otherwise, it is not written.
charsetQ_CHARSET_CODE == 4three 2 byte unsigned integers, totally 2+2+2=6 bytesThe three variables character_set_client, collation_connection, and + * collation_server, in that order. character_set_client is a code identifying + * the character set and collation used by the client to encode the query. + * collation_connection identifies the character set and collation that the + * master converts the query to when it receives it; this is useful when + * comparing literal strings. collation_server is the default character set and + * collation used when a new database is created. + * + * See also "Connection Character Sets and Collations" in the MySQL 5.1 manual. + * + * All three variables are codes identifying a (character set, collation) pair. + * To see which codes map to which pairs, run the query "SELECT id, + * character_set_name, collation_name FROM COLLATIONS". + * + * Cf. Q_CHARSET_DATABASE_CODE below. + * + * This field is always written.
time_zoneQ_TIME_ZONE_CODE == 5Variable-length string: the length in bytes (1 byte) followed by the + * characters (at most 255 bytes). + * The time_zone of the master. + * + * See also "System Variables" and "MySQL Server Time Zone Support" in the MySQL + * manual. + * + * This field is written if the length of the time zone string is > 0; + * otherwise, it is not written.
lc_time_names_numberQ_LC_TIME_NAMES_CODE == 72 byte integerA code identifying a table of month and day names. The mapping from codes + * to languages is defined in sql_locale.cc. This field is written if it is not + * 0, i.e., if the locale is not en_US.
charset_database_numberQ_CHARSET_DATABASE_CODE == 82 byte integerThe value of the collation_database system variable (in the source code + * stored in thd->variables.collation_database), which holds the code for a + * (character set, collation) pair as described above (see Q_CHARSET_CODE). + * + * collation_database was used in old versions (???WHEN). Its value was loaded + * when issuing a "use db" query and could be changed by issuing a + * "SET collation_database=xxx" query. It used to affect the "LOAD DATA INFILE" + * and "CREATE TABLE" commands. + * + * In newer versions, "CREATE TABLE" has been changed to take the character set + * from the database of the created table, rather than the character set of the + * current database. This makes a difference when creating a table in another + * database than the current one. "LOAD DATA INFILE" has not yet changed to do + * this, but there are plans to eventually do it, and to make collation_database + * read-only. + * + * This field is written if it is not 0.
table_map_for_updateQ_TABLE_MAP_FOR_UPDATE_CODE == 98 byte integerThe value of the table map that is to be updated by the multi-table + * update query statement. Every bit of this variable represents a table, and is + * set to 1 if the corresponding table is to be updated by this statement. + * + * The value of this variable is set when executing a multi-table update + * statement and used by slave to apply filter rules without opening all the + * tables on slave. This is required because some tables may not exist on slave + * because of the filter rules.
+ * + * Query_log_event_notes_on_previous_versions Notes on Previous Versions + * + * Status vars were introduced in version 5.0. To read earlier versions + * correctly, check the length of the Post-Header. + * + * The status variable Q_CATALOG_CODE == 2 existed in MySQL 5.0.x, where + * 0<=x<=3. It was identical to Q_CATALOG_CODE, except that the string had a + * trailing '\0'. The '\0' was removed in 5.0.4 since it was redundant (the + * string length is stored before the string). The Q_CATALOG_CODE will never be + * written by a new master, but can still be understood by a new slave. + * + * See Q_CHARSET_DATABASE_CODE in the table above. + * + * When adding new status vars, please don't forget to update the + * MAX_SIZE_LOG_EVENT_STATUS, and update function code_name + * + * @see mysql-5.1.6/sql/logevent.cc - Query_log_event + * + * @author Changyuan.lh + * @version 1.0 + */ +public class QueryLogEvent extends LogEvent +{ + /** + The maximum number of updated databases that a status of + Query-log-event can carry. It can redefined within a range + [1.. OVER_MAX_DBS_IN_EVENT_MTS]. + */ + public static final int MAX_DBS_IN_EVENT_MTS = 16; + + /** + When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS + the value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the + mts_accessed_dbs status. + */ + public static final int OVER_MAX_DBS_IN_EVENT_MTS = 254; + + + public static final int SYSTEM_CHARSET_MBMAXLEN = 3; + public static final int NAME_CHAR_LEN = 64; + /* Field/table name length */ + public static final int NAME_LEN = (NAME_CHAR_LEN * SYSTEM_CHARSET_MBMAXLEN); + + /** + * Max number of possible extra bytes in a replication event compared to a + * packet (i.e. a query) sent from client to master; First, an auxiliary + * log_event status vars estimation: + */ + public static final int MAX_SIZE_LOG_EVENT_STATUS = (1 + 4 /* type, flags2 */ + + 1 + 8 /* type, sql_mode */ + + 1 + 1 + 255 /* type, length, catalog */ + + 1 + 4 /* type, auto_increment */ + + 1 + 6 /* type, charset */ + + 1 + 1 + 255 /* type, length, time_zone */ + + 1 + 2 /* type, lc_time_names_number */ + + 1 + 2 /* type, charset_database_number */ + + 1 + 8 /* type, table_map_for_update */ + + 1 + 4 /* type, master_data_written */ + /* type, db_1, db_2, ... */ + /* type, microseconds */ + /* MariaDb type, sec_part of NOW() */ + + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3 + + 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */); + /** + * Fixed data part: + * + *
    + *
  • 4 bytes. The ID of the thread that issued this statement. Needed for + * temporary tables. This is also useful for a DBA for knowing who did what + * on the master.
    • + *
  • 4 bytes. The time in seconds that the statement took to execute. Only + * useful for inspection by the DBA.
    • + *
  • 1 byte. The length of the name of the database which was the default + * database when the statement was executed. This name appears later, in the + * variable data part. It is necessary for statements such as INSERT INTO t + * VALUES(1) that don't specify the database and rely on the default + * database previously selected by USE.
    • + *
  • 2 bytes. The error code resulting from execution of the statement on + * the master. Error codes are defined in include/mysqld_error.h. 0 means no + * error. How come statements with a non-zero error code can exist in the + * binary log? This is mainly due to the use of non-transactional tables + * within transactions. For example, if an INSERT ... SELECT fails after + * inserting 1000 rows into a MyISAM table (for example, with a + * duplicate-key violation), we have to write this statement to the binary + * log, because it truly modified the MyISAM table. For transactional + * tables, there should be no event with a non-zero error code (though it + * can happen, for example if the connection was interrupted (Control-C)). + * The slave checks the error code: After executing the statement itself, it + * compares the error code it got with the error code in the event, and if + * they are different it stops replicating (unless --slave-skip-errors was + * used to ignore the error).
    • + *
  • 2 bytes (not present in v1, v3). The length of the status variable + * block.
    • + *
+ * + * Variable part: + *
    + *
  • Zero or more status variables (not present in v1, v3). Each status + * variable consists of one byte code identifying the variable stored, + * followed by the value of the variable. The format of the value is + * variable-specific, as described later.
    • + *
  • The default database name (null-terminated).
    • + *
  • The SQL statement. The slave knows the size of the other fields in + * the variable part (the sizes are given in the fixed data part), so by + * subtraction it can know the size of the statement.
    • + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + private String user; + private String host; + + /* using byte for query string */ + protected String query; + protected String catalog; + protected final String dbname; + + /** The number of seconds the query took to run on the master. */ + // The time in seconds that the statement took to execute. Only useful for inspection by the DBA + private final long execTime; + private final int errorCode; + private final long sessionId; /* thread_id */ + + /** + * 'flags2' is a second set of flags (on top of those in Log_event), for + * session variables. These are thd->options which is & against a mask + * (OPTIONS_WRITTEN_TO_BIN_LOG). + */ + private long flags2; + + /** In connections sql_mode is 32 bits now but will be 64 bits soon */ + private long sql_mode; + + private long autoIncrementIncrement = -1; + private long autoIncrementOffset = -1; + + private int clientCharset = -1; + private int clientCollation = -1; + private int serverCollation = -1; + private String charsetName; + + private String timezone; + + public QueryLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) throws IOException + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; + /* + * We test if the event's length is sensible, and if so we compute + * data_len. We cannot rely on QUERY_HEADER_LEN here as it would not be + * format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same + * for 3.23, 4.0 & 5.0. + */ + if (buffer.limit() < (commonHeaderLen + postHeaderLen)) + { + throw new IOException("Query event length is too short."); + } + int dataLen = buffer.limit() - (commonHeaderLen + postHeaderLen); + buffer.position(commonHeaderLen + Q_THREAD_ID_OFFSET); + + sessionId = buffer.getUint32(); // Q_THREAD_ID_OFFSET + execTime = buffer.getUint32(); // Q_EXEC_TIME_OFFSET + + // TODO: add a check of all *_len vars + final int dbLen = buffer.getUint8(); // Q_DB_LEN_OFFSET + errorCode = buffer.getUint16(); // Q_ERR_CODE_OFFSET + + /* + * 5.0 format starts here. Depending on the format, we may or not + * have affected/warnings etc The remaining post-header to be parsed + * has length: + */ + int statusVarsLen = 0; + if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN) + { + statusVarsLen = buffer.getUint16(); // Q_STATUS_VARS_LEN_OFFSET + /* + Check if status variable length is corrupt and will lead to very + wrong data. We could be even more strict and require data_len to + be even bigger, but this will suffice to catch most corruption + errors that can lead to a crash. + */ + if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS)) + { + throw new IOException("status_vars_len (" + statusVarsLen + + ") > data_len (" + dataLen + ")"); + } + dataLen -= statusVarsLen; + } + /* + * We have parsed everything we know in the post header for QUERY_EVENT, + * the rest of post header is either comes from older version MySQL or + * dedicated to derived events (e.g. Execute_load_query...) + */ + + /* variable-part: the status vars; only in MySQL 5.0 */ + final int start = commonHeaderLen + postHeaderLen; + final int limit = buffer.limit(); /* for restore */ + final int end = start + statusVarsLen; + buffer.position(start).limit(end); + unpackVariables(buffer, end); + buffer.position(end); + buffer.limit(limit); + + /* A 2nd variable part; this is common to all versions */ + final int queryLen = dataLen - dbLen - 1; + dbname = buffer.getFixString(dbLen + 1); + if (clientCharset >= 0) + { + charsetName = CharsetConversion.getJavaCharset(clientCharset); + + if ((charsetName != null) && (Charset.isSupported(charsetName))) + { + query = buffer.getFixString(queryLen, charsetName); + } + else + { + logger.warn("unsupported character set in query log: " + + "\n ID = " + clientCharset + ", Charset = " + + CharsetConversion.getCharset(clientCharset) + + ", Collation = " + + CharsetConversion.getCollation(clientCharset)); + + query = buffer.getFixString(queryLen); + } + } + else + { + query = buffer.getFixString(queryLen); + } + } + + /* query event post-header */ + public static final int Q_THREAD_ID_OFFSET = 0; + public static final int Q_EXEC_TIME_OFFSET = 4; + public static final int Q_DB_LEN_OFFSET = 8; + public static final int Q_ERR_CODE_OFFSET = 9; + public static final int Q_STATUS_VARS_LEN_OFFSET = 11; + public static final int Q_DATA_OFFSET = QUERY_HEADER_LEN; + + /* these are codes, not offsets; not more than 256 values (1 byte). */ + public static final int Q_FLAGS2_CODE = 0; + public static final int Q_SQL_MODE_CODE = 1; + + /** + * Q_CATALOG_CODE is catalog with end zero stored; it is used only by MySQL + * 5.0.x where 0<=x<=3. We have to keep it to be able to replicate these old + * masters. + */ + public static final int Q_CATALOG_CODE = 2; + public static final int Q_AUTO_INCREMENT = 3; + public static final int Q_CHARSET_CODE = 4; + public static final int Q_TIME_ZONE_CODE = 5; + + /** + * Q_CATALOG_NZ_CODE is catalog withOUT end zero stored; it is used by MySQL + * 5.0.x where x>=4. Saves one byte in every Query_log_event in binlog, + * compared to Q_CATALOG_CODE. The reason we didn't simply re-use + * Q_CATALOG_CODE is that then a 5.0.3 slave of this 5.0.x (x>=4) master + * would crash (segfault etc) because it would expect a 0 when there is + * none. + */ + public static final int Q_CATALOG_NZ_CODE = 6; + + public static final int Q_LC_TIME_NAMES_CODE = 7; + + public static final int Q_CHARSET_DATABASE_CODE = 8; + + public static final int Q_TABLE_MAP_FOR_UPDATE_CODE = 9; + + public static final int Q_MASTER_DATA_WRITTEN_CODE = 10; + + public static final int Q_INVOKER = 11; + + /** + Q_UPDATED_DB_NAMES status variable collects of the updated databases + total number and their names to be propagated to the slave in order + to facilitate the parallel applying of the Query events. + */ + public static final int Q_UPDATED_DB_NAMES = 12; + + public static final int Q_MICROSECONDS = 13; + + /** + * FROM MariaDB 5.5.34 + */ + public static final int Q_HRNOW = 128; + + private final void unpackVariables(LogBuffer buffer, final int end) + throws IOException + { + int code = -1; + try + { + while (buffer.position() < end) + { + switch (code = buffer.getUint8()) + { + case Q_FLAGS2_CODE: + flags2 = buffer.getUint32(); + break; + case Q_SQL_MODE_CODE: + sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode is ulonglong + break; + case Q_CATALOG_NZ_CODE: + catalog = buffer.getString(); + break; + case Q_AUTO_INCREMENT: + autoIncrementIncrement = buffer.getUint16(); + autoIncrementOffset = buffer.getUint16(); + break; + case Q_CHARSET_CODE: + // Charset: 6 byte character set flag. + // 1-2 = character set client + // 3-4 = collation client + // 5-6 = collation server + clientCharset = buffer.getUint16(); + clientCollation = buffer.getUint16(); + serverCollation = buffer.getUint16(); + break; + case Q_TIME_ZONE_CODE: + timezone = buffer.getString(); + break; + case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */ + final int len = buffer.getUint8(); + catalog = buffer.getFixString(len + 1); + break; + case Q_LC_TIME_NAMES_CODE: + // lc_time_names_number = buffer.getUint16(); + buffer.forward(2); + break; + case Q_CHARSET_DATABASE_CODE: + // charset_database_number = buffer.getUint16(); + buffer.forward(2); + break; + case Q_TABLE_MAP_FOR_UPDATE_CODE: + // table_map_for_update = buffer.getUlong64(); + buffer.forward(8); + break; + case Q_MASTER_DATA_WRITTEN_CODE: + // data_written = master_data_written = buffer.getUint32(); + buffer.forward(4); + break; + case Q_INVOKER: + user = buffer.getString(); + host = buffer.getString(); + break; + case Q_MICROSECONDS: + // when.tv_usec= uint3korr(pos); + buffer.forward(3); + break; + case Q_UPDATED_DB_NAMES: + int mtsAccessedDbs = buffer.getUint8(); + /** + Notice, the following check is positive also in case of + the master's MAX_DBS_IN_EVENT_MTS > the slave's one and the event + contains e.g the master's MAX_DBS_IN_EVENT_MTS db:s. + */ + if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) { + mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS; + break; + } + String mtsAccessedDbNames[] = new String[mtsAccessedDbs]; + for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) { + int length = end - buffer.position(); + mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN); + } + break; + case Q_HRNOW: + // int when_sec_part = buffer.getUint24(); + buffer.forward(3); + break; + default: + /* That's why you must write status vars in growing order of code */ + if (logger.isDebugEnabled()) + logger.debug("Query_log_event has unknown status vars (first has code: " + + code + "), skipping the rest of them"); + break; // Break loop + } + } + } + catch (RuntimeException e) + { + throw new IOException("Read " + findCodeName(code) + " error: " + + e.getMessage(), e); + } + } + + private static final String findCodeName(final int code) + { + switch (code) + { + case Q_FLAGS2_CODE: + return "Q_FLAGS2_CODE"; + case Q_SQL_MODE_CODE: + return "Q_SQL_MODE_CODE"; + case Q_CATALOG_CODE: + return "Q_CATALOG_CODE"; + case Q_AUTO_INCREMENT: + return "Q_AUTO_INCREMENT"; + case Q_CHARSET_CODE: + return "Q_CHARSET_CODE"; + case Q_TIME_ZONE_CODE: + return "Q_TIME_ZONE_CODE"; + case Q_CATALOG_NZ_CODE: + return "Q_CATALOG_NZ_CODE"; + case Q_LC_TIME_NAMES_CODE: + return "Q_LC_TIME_NAMES_CODE"; + case Q_CHARSET_DATABASE_CODE: + return "Q_CHARSET_DATABASE_CODE"; + case Q_TABLE_MAP_FOR_UPDATE_CODE: + return "Q_TABLE_MAP_FOR_UPDATE_CODE"; + case Q_MASTER_DATA_WRITTEN_CODE: + return "Q_MASTER_DATA_WRITTEN_CODE"; + case Q_UPDATED_DB_NAMES: + return "Q_UPDATED_DB_NAMES"; + case Q_MICROSECONDS: + return "Q_MICROSECONDS"; + } + return "CODE#" + code; + } + + public final String getUser() + { + return user; + } + + public final String getHost() + { + return host; + } + + public final String getQuery() + { + return query; + } + + public final String getCatalog() + { + return catalog; + } + + public final String getDbName() + { + return dbname; + } + + /** + * The number of seconds the query took to run on the master. + */ + public final long getExecTime() + { + return execTime; + } + + public final int getErrorCode() + { + return errorCode; + } + + public final long getSessionId() + { + return sessionId; + } + + public final long getAutoIncrementIncrement() + { + return autoIncrementIncrement; + } + + public final long getAutoIncrementOffset() + { + return autoIncrementOffset; + } + + public final String getCharsetName() + { + return charsetName; + } + + public final String getTimezone() + { + return timezone; + } + + /** + * Returns the charsetID value. + * + * @return Returns the charsetID. + */ + public final int getClientCharset() + { + return clientCharset; + } + + /** + * Returns the clientCollationId value. + * + * @return Returns the clientCollationId. + */ + public final int getClientCollation() + { + return clientCollation; + } + + /** + * Returns the serverCollationId value. + * + * @return Returns the serverCollationId. + */ + public final int getServerCollation() + { + return serverCollation; + } + + /** + * Returns the sql_mode value. + * + *

+ * The sql_mode variable. See the section "SQL Modes" in the MySQL manual, + * and see mysql_priv.h for a list of the possible flags. Currently + * (2007-10-04), the following flags are available: + * + *

    + *
  • MODE_REAL_AS_FLOAT==0x1
    • + *
  • MODE_PIPES_AS_CONCAT==0x2
    • + *
  • MODE_ANSI_QUOTES==0x4
    • + *
  • MODE_IGNORE_SPACE==0x8
    • + *
  • MODE_NOT_USED==0x10
    • + *
  • MODE_ONLY_FULL_GROUP_BY==0x20
    • + *
  • MODE_NO_UNSIGNED_SUBTRACTION==0x40
    • + *
  • MODE_NO_DIR_IN_CREATE==0x80
    • + *
  • MODE_POSTGRESQL==0x100
    • + *
  • MODE_ORACLE==0x200
    • + *
  • MODE_MSSQL==0x400
    • + *
  • MODE_DB2==0x800
    • + *
  • MODE_MAXDB==0x1000
    • + *
  • MODE_NO_KEY_OPTIONS==0x2000
    • + *
  • MODE_NO_TABLE_OPTIONS==0x4000
    • + *
  • MODE_NO_FIELD_OPTIONS==0x8000
    • + *
  • MODE_MYSQL323==0x10000
    • + *
  • MODE_MYSQL40==0x20000
    • + *
  • MODE_ANSI==0x40000
    • + *
  • MODE_NO_AUTO_VALUE_ON_ZERO==0x80000
    • + *
  • MODE_NO_BACKSLASH_ESCAPES==0x100000
    • + *
  • MODE_STRICT_TRANS_TABLES==0x200000
    • + *
  • MODE_STRICT_ALL_TABLES==0x400000
    • + *
  • MODE_NO_ZERO_IN_DATE==0x800000
    • + *
  • MODE_NO_ZERO_DATE==0x1000000
    • + *
  • MODE_INVALID_DATES==0x2000000
    • + *
  • MODE_ERROR_FOR_DIVISION_BY_ZERO==0x4000000
    • + *
  • MODE_TRADITIONAL==0x8000000
    • + *
  • MODE_NO_AUTO_CREATE_USER==0x10000000
    • + *
  • MODE_HIGH_NOT_PRECEDENCE==0x20000000
    • + *
  • MODE_NO_ENGINE_SUBSTITUTION=0x40000000
    • + *
  • MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000
    • + *
+ * + * All these flags are replicated from the server. However, all flags except + * MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always + * preserves its old value of MODE_NO_DIR_IN_CREATE. This field is always + * written to the binlog. + */ + public final long getSqlMode() + { + return sql_mode; + } + + /* FLAGS2 values that can be represented inside the binlog */ + public static final int OPTION_AUTO_IS_NULL = 1 << 14; + public static final int OPTION_NOT_AUTOCOMMIT = 1 << 19; + public static final int OPTION_NO_FOREIGN_KEY_CHECKS = 1 << 26; + public static final int OPTION_RELAXED_UNIQUE_CHECKS = 1 << 27; + + /** + * The flags in thd->options, binary AND-ed with OPTIONS_WRITTEN_TO_BIN_LOG. + * The thd->options bitfield contains options for "SELECT". OPTIONS_WRITTEN + * identifies those options that need to be written to the binlog (not all + * do). Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL + * | OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS | + * OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. These flags correspond to + * the SQL variables SQL_AUTO_IS_NULL, FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, + * and AUTOCOMMIT, documented in the "SET Syntax" section of the MySQL + * Manual. This field is always written to the binlog in version >= 5.0, and + * never written in version < 5.0. + */ + public final long getFlags2() + { + return flags2; + } + + /** + * Returns the OPTION_AUTO_IS_NULL flag. + */ + public final boolean isAutoIsNull() + { + return ((flags2 & OPTION_AUTO_IS_NULL) == OPTION_AUTO_IS_NULL); + } + + /** + * Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag. + */ + public final boolean isForeignKeyChecks() + { + return ((flags2 & OPTION_NO_FOREIGN_KEY_CHECKS) != OPTION_NO_FOREIGN_KEY_CHECKS); + } + + /** + * Returns the OPTION_NOT_AUTOCOMMIT flag. + */ + public final boolean isAutocommit() + { + return ((flags2 & OPTION_NOT_AUTOCOMMIT) != OPTION_NOT_AUTOCOMMIT); + } + + /** + * Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag. + */ + public final boolean isUniqueChecks() + { + return ((flags2 & OPTION_RELAXED_UNIQUE_CHECKS) != OPTION_RELAXED_UNIQUE_CHECKS); + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java new file mode 100644 index 00000000..7fa36085 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java @@ -0,0 +1,83 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1 + * does not need it (it's repeatable again) so this event needn't be written in + * 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it + * does not cause bugs). + * + * The state of the random number generation consists of 128 bits, which are + * stored internally as two 64-bit numbers. + * + * Binary Format + * + * The Post-Header for this event type is empty. The Body has two components: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Body for Rand_log_event
NameFormatDescription
seed18 byte unsigned integer64 bit random seed1.
seed28 byte unsigned integer64 bit random seed2.
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class RandLogEvent extends LogEvent +{ + /** + * Fixed data part: Empty + * + *

+ * Variable data part: + * + *

    + *
  • 8 bytes. The value for the first seed.
    • + *
  • 8 bytes. The value for the second seed.
    • + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + private final long seed1; + private final long seed2; + + /* Rand event data */ + public static final int RAND_SEED1_OFFSET = 0; + public static final int RAND_SEED2_OFFSET = 8; + + public RandLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + /* The Post-Header is empty. The Variable Data part begins immediately. */ + buffer.position(descriptionEvent.commonHeaderLen + + descriptionEvent.postHeaderLen[RAND_EVENT - 1] + + RAND_SEED1_OFFSET); + seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET); + seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET); + } + + public final String getQuery() + { + return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java new file mode 100644 index 00000000..411cf2a6 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java @@ -0,0 +1,145 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * This will be deprecated when we move to using sequence ids. + * + * Binary Format + * + * The Post-Header has one component: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Post-Header for Rotate_log_event
NameFormatDescription
position8 byte integerThe position within the binlog to rotate to.
+ * + * The Body has one component: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Body for Rotate_log_event
NameFormatDescription
new_logvariable length string without trailing zero, extending to the end of the + * event (determined by the length field of the Common-Header)Name of the binlog to rotate to.
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class RotateLogEvent extends LogEvent +{ + /** + * Fixed data part: + * + *
    + *
  • 8 bytes. The position of the first event in the next log file. Always + * contains the number 4 (meaning the next event starts at position 4 in the + * next binary log). This field is not present in v1; presumably the value + * is assumed to be 4.
    • + *
+ *

+ * + * Variable data part: + * + *

    + *
  • The name of the next binary log. The filename is not null-terminated. + * Its length is the event size minus the size of the fixed parts.
    • + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + private final String filename; + private final long position; + + /* Rotate event post-header */ + public static final int R_POS_OFFSET = 0; + public static final int R_IDENT_OFFSET = 8; + + /* Max length of full path-name */ + public static final int FN_REFLEN = 512; + + // Rotate header with all empty fields. + public static final LogHeader ROTATE_HEADER = new LogHeader(ROTATE_EVENT); + + /** + * Creates a new Rotate_log_event object read normally from + * log. + * + * @throws MySQLExtractException + */ + public RotateLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int headerSize = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1]; + + buffer.position(headerSize + R_POS_OFFSET); + position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET) + + final int filenameOffset = headerSize + postHeaderLen; + int filenameLen = buffer.limit() - filenameOffset; + if (filenameLen > FN_REFLEN - 1) + filenameLen = FN_REFLEN - 1; + buffer.position(filenameOffset); + filename = buffer.getFixString(filenameLen); + } + + /** + * Creates a new Rotate_log_event without log information. This + * is used to generate missing log rotation events. + */ + public RotateLogEvent(String filename) + { + super(ROTATE_HEADER); + + this.filename = filename; + this.position = 4; + } + + /** + * Creates a new Rotate_log_event without log information. + */ + public RotateLogEvent(String filename, final long position) + { + super(ROTATE_HEADER); + + this.filename = filename; + this.position = position; + } + + public final String getFilename() + { + return filename; + } + + public final long getPosition() + { + return position; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java new file mode 100644 index 00000000..908e4f59 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java @@ -0,0 +1,976 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.io.Serializable; +import java.sql.Timestamp; +import java.sql.Types; +import java.util.BitSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Extracting JDBC type & value information from packed rows-buffer. + * + * @see mysql-5.1.60/sql/log_event.cc - Rows_log_event::print_verbose_one_row + * @author Changyuan.lh + * @version 1.0 + */ +public final class RowsLogBuffer { + + protected static final Log logger = LogFactory.getLog(RowsLogBuffer.class); + + public static final long DATETIMEF_INT_OFS = 0x8000000000L; + public static final long TIMEF_INT_OFS = 0x800000L; + private final LogBuffer buffer; + private final int columnLen; + private final String charsetName; + // private Calendar cal; + + private final BitSet nullBits; + private int nullBitIndex; + + private boolean fNull; + private int javaType; + private int length; + private Serializable value; + + public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName){ + this.buffer = buffer; + this.columnLen = columnLen; + this.charsetName = charsetName; + this.nullBits = new BitSet(columnLen); + } + + /** + * Extracting next row from packed buffer. + * + * @see mysql-5.1.60/sql/log_event.cc - + * Rows_log_event::print_verbose_one_row + */ + public final boolean nextOneRow(BitSet columns) { + final boolean hasOneRow = buffer.hasRemaining(); + + if (hasOneRow) { + int column = 0; + + for (int i = 0; i < columnLen; i++) + if (columns.get(i)) column++; + + nullBitIndex = 0; + nullBits.clear(); + buffer.fillBitmap(nullBits, column); + } + return hasOneRow; + } + + /** + * Extracting next field value from packed buffer. + * + * @see mysql-5.1.60/sql/log_event.cc - + * Rows_log_event::print_verbose_one_row + */ + public final Serializable nextValue(final int type, final int meta) { + return nextValue(type, meta, false); + } + + /** + * Extracting next field value from packed buffer. + * + * @see mysql-5.1.60/sql/log_event.cc - + * Rows_log_event::print_verbose_one_row + */ + public final Serializable nextValue(final int type, final int meta, boolean isBinary) { + fNull = nullBits.get(nullBitIndex++); + + if (fNull) { + value = null; + javaType = mysqlToJavaType(type, meta, isBinary); + length = 0; + return null; + } else { + // Extracting field value from packed buffer. + return fetchValue(type, meta, isBinary); + } + } + + /** + * Maps the given MySQL type to the correct JDBC type. + */ + static int mysqlToJavaType(int type, final int meta, boolean isBinary) { + int javaType; + + if (type == LogEvent.MYSQL_TYPE_STRING) { + if (meta >= 256) { + int byte0 = meta >> 8; + if ((byte0 & 0x30) != 0x30) { + /* a long CHAR() field: see #37426 */ + type = byte0 | 0x30; + } else { + switch (byte0) { + case LogEvent.MYSQL_TYPE_SET: + case LogEvent.MYSQL_TYPE_ENUM: + case LogEvent.MYSQL_TYPE_STRING: + type = byte0; + } + } + } + } + + switch (type) { + case LogEvent.MYSQL_TYPE_LONG: + javaType = Types.INTEGER; + break; + + case LogEvent.MYSQL_TYPE_TINY: + javaType = Types.TINYINT; + break; + + case LogEvent.MYSQL_TYPE_SHORT: + javaType = Types.SMALLINT; + break; + + case LogEvent.MYSQL_TYPE_INT24: + javaType = Types.INTEGER; + break; + + case LogEvent.MYSQL_TYPE_LONGLONG: + javaType = Types.BIGINT; + break; + + case LogEvent.MYSQL_TYPE_DECIMAL: + javaType = Types.DECIMAL; + break; + + case LogEvent.MYSQL_TYPE_NEWDECIMAL: + javaType = Types.DECIMAL; + break; + + case LogEvent.MYSQL_TYPE_FLOAT: + javaType = Types.REAL; // Types.FLOAT; + break; + + case LogEvent.MYSQL_TYPE_DOUBLE: + javaType = Types.DOUBLE; + break; + + case LogEvent.MYSQL_TYPE_BIT: + javaType = Types.BIT; + break; + + case LogEvent.MYSQL_TYPE_TIMESTAMP: + case LogEvent.MYSQL_TYPE_DATETIME: + javaType = Types.TIMESTAMP; + break; + + case LogEvent.MYSQL_TYPE_TIME: + javaType = Types.TIME; + break; + + case LogEvent.MYSQL_TYPE_NEWDATE: + case LogEvent.MYSQL_TYPE_DATE: + javaType = Types.DATE; + break; + + case LogEvent.MYSQL_TYPE_YEAR: + javaType = Types.VARCHAR; + break; + + case LogEvent.MYSQL_TYPE_ENUM: + javaType = Types.INTEGER; + break; + + case LogEvent.MYSQL_TYPE_SET: + javaType = Types.BINARY; + break; + + case LogEvent.MYSQL_TYPE_TINY_BLOB: + case LogEvent.MYSQL_TYPE_MEDIUM_BLOB: + case LogEvent.MYSQL_TYPE_LONG_BLOB: + case LogEvent.MYSQL_TYPE_BLOB: + if (meta == 1) { + javaType = Types.VARBINARY; + } else { + javaType = Types.LONGVARBINARY; + } + break; + + case LogEvent.MYSQL_TYPE_VARCHAR: + case LogEvent.MYSQL_TYPE_VAR_STRING: + if (isBinary) { + // varbinary在binlog中为var_string类型 + javaType = Types.VARBINARY; + } else { + javaType = Types.VARCHAR; + } + break; + + case LogEvent.MYSQL_TYPE_STRING: + if (isBinary) { + // binary在binlog中为string类型 + javaType = Types.BINARY; + } else { + javaType = Types.CHAR; + } + break; + + case LogEvent.MYSQL_TYPE_GEOMETRY: + javaType = Types.BINARY; + break; + + // case LogEvent.MYSQL_TYPE_BINARY: + // javaType = Types.BINARY; + // break; + // + // case LogEvent.MYSQL_TYPE_VARBINARY: + // javaType = Types.VARBINARY; + // break; + + default: + javaType = Types.OTHER; + } + + return javaType; + } + + /** + * Extracting next field value from packed buffer. + * + * @see mysql-5.1.60/sql/log_event.cc - log_event_print_value + */ + final Serializable fetchValue(int type, final int meta, boolean isBinary) { + int len = 0; + + if (type == LogEvent.MYSQL_TYPE_STRING) { + if (meta >= 256) { + int byte0 = meta >> 8; + int byte1 = meta & 0xff; + if ((byte0 & 0x30) != 0x30) { + /* a long CHAR() field: see #37426 */ + len = byte1 | (((byte0 & 0x30) ^ 0x30) << 4); + type = byte0 | 0x30; + } else { + switch (byte0) { + case LogEvent.MYSQL_TYPE_SET: + case LogEvent.MYSQL_TYPE_ENUM: + case LogEvent.MYSQL_TYPE_STRING: + type = byte0; + len = byte1; + break; + default: + throw new IllegalArgumentException(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)", + type, + meta, + meta)); + } + } + } else { + len = meta; + } + } + + switch (type) { + case LogEvent.MYSQL_TYPE_LONG: { + // XXX: How to check signed / unsigned? + // value = unsigned ? Long.valueOf(buffer.getUint32()) : + // Integer.valueOf(buffer.getInt32()); + value = Integer.valueOf(buffer.getInt32()); + javaType = Types.INTEGER; + length = 4; + break; + } + case LogEvent.MYSQL_TYPE_TINY: { + // XXX: How to check signed / unsigned? + // value = Integer.valueOf(unsigned ? buffer.getUint8() : + // buffer.getInt8()); + value = Integer.valueOf(buffer.getInt8()); + javaType = Types.TINYINT; // java.sql.Types.INTEGER; + length = 1; + break; + } + case LogEvent.MYSQL_TYPE_SHORT: { + // XXX: How to check signed / unsigned? + // value = Integer.valueOf(unsigned ? buffer.getUint16() : + // buffer.getInt16()); + value = Integer.valueOf((short) buffer.getInt16()); + javaType = Types.SMALLINT; // java.sql.Types.INTEGER; + length = 2; + break; + } + case LogEvent.MYSQL_TYPE_INT24: { + // XXX: How to check signed / unsigned? + // value = Integer.valueOf(unsigned ? buffer.getUint24() : + // buffer.getInt24()); + value = Integer.valueOf(buffer.getInt24()); + javaType = Types.INTEGER; + length = 3; + break; + } + case LogEvent.MYSQL_TYPE_LONGLONG: { + // XXX: How to check signed / unsigned? + // value = unsigned ? buffer.getUlong64()) : + // Long.valueOf(buffer.getLong64()); + value = Long.valueOf(buffer.getLong64()); + javaType = Types.BIGINT; // Types.INTEGER; + length = 8; + break; + } + case LogEvent.MYSQL_TYPE_DECIMAL: { + /* + * log_event.h : This enumeration value is only used internally + * and cannot exist in a binlog. + */ + logger.warn("MYSQL_TYPE_DECIMAL : This enumeration value is " + + "only used internally and cannot exist in a binlog!"); + javaType = Types.DECIMAL; + value = null; /* unknown format */ + length = 0; + break; + } + case LogEvent.MYSQL_TYPE_NEWDECIMAL: { + final int precision = meta >> 8; + final int decimals = meta & 0xff; + value = buffer.getDecimal(precision, decimals); + javaType = Types.DECIMAL; + length = precision; + break; + } + case LogEvent.MYSQL_TYPE_FLOAT: { + value = Float.valueOf(buffer.getFloat32()); + javaType = Types.REAL; // Types.FLOAT; + length = 4; + break; + } + case LogEvent.MYSQL_TYPE_DOUBLE: { + value = Double.valueOf(buffer.getDouble64()); + javaType = Types.DOUBLE; + length = 8; + break; + } + case LogEvent.MYSQL_TYPE_BIT: { + /* Meta-data: bit_len, bytes_in_rec, 2 bytes */ + final int nbits = ((meta >> 8) * 8) + (meta & 0xff); + len = (nbits + 7) / 8; + if (nbits > 1) { + // byte[] bits = new byte[len]; + // buffer.fillBytes(bits, 0, len); + // 转化为unsign long + switch (len) { + case 1: + value = buffer.getInt8(); + break; + case 2: + value = buffer.getBeUint16(); + break; + case 3: + value = buffer.getBeUint24(); + break; + case 4: + value = buffer.getBeUint32(); + break; + case 5: + value = buffer.getBeUlong40(); + break; + case 6: + value = buffer.getBeUlong48(); + break; + case 7: + value = buffer.getBeUlong56(); + break; + case 8: + value = buffer.getBeUlong64(); + break; + default: + throw new IllegalArgumentException("!! Unknown Bit len = " + len); + } + } else { + final int bit = buffer.getInt8(); + // value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE; + value = bit; + } + javaType = Types.BIT; + length = nbits; + break; + } + case LogEvent.MYSQL_TYPE_TIMESTAMP: { + // MYSQL DataTypes: TIMESTAMP + // range is '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' + // UTC + // A TIMESTAMP cannot represent the value '1970-01-01 00:00:00' + // because that is equivalent to 0 seconds from the epoch and + // the value 0 is reserved for representing '0000-00-00 + // 00:00:00', the “zero” TIMESTAMP value. + final long i32 = buffer.getUint32(); + if (i32 == 0) { + value = "0000-00-00 00:00:00"; + } else { + String v = new Timestamp(i32 * 1000).toString(); + value = v.substring(0, v.length() - 2); + } + javaType = Types.TIMESTAMP; + length = 4; + break; + } + case LogEvent.MYSQL_TYPE_TIMESTAMP2: { + final long tv_sec = buffer.getBeUint32(); // big-endian + int tv_usec = 0; + switch (meta) { + case 0: + tv_usec = 0; + break; + case 1: + case 2: + tv_usec = buffer.getInt8() * 10000; + break; + case 3: + case 4: + tv_usec = buffer.getBeInt16() * 100; + break; + case 5: + case 6: + tv_usec = buffer.getBeInt24(); + break; + default: + tv_usec = 0; + break; + } + + if (tv_sec == 0) { + value = "0000-00-00 00:00:00"; + } else { + Timestamp time = new Timestamp(tv_sec * 1000); + time.setNanos(tv_usec * 1000); + String v = time.toString(); + value = v.substring(0, v.length() - 2); + } + javaType = Types.TIMESTAMP; + length = 4 + (meta + 1) / 2; + break; + } + case LogEvent.MYSQL_TYPE_DATETIME: { + // MYSQL DataTypes: DATETIME + // range is '0000-01-01 00:00:00' to '9999-12-31 23:59:59' + final long i64 = buffer.getLong64(); /* YYYYMMDDhhmmss */ + if (i64 == 0) { + value = "0000-00-00 00:00:00"; + } else { + final int d = (int) (i64 / 1000000); + final int t = (int) (i64 % 1000000); + // if (cal == null) cal = Calendar.getInstance(); + // cal.clear(); + /* month is 0-based, 0 for january. */ + // cal.set(d / 10000, (d % 10000) / 100 - 1, d % 100, t / + // 10000, (t % 10000) / 100, t % 100); + // value = new Timestamp(cal.getTimeInMillis()); + value = String.format("%04d-%02d-%02d %02d:%02d:%02d", + d / 10000, + (d % 10000) / 100, + d % 100, + t / 10000, + (t % 10000) / 100, + t % 100); + } + javaType = Types.TIMESTAMP; + length = 8; + break; + } + case LogEvent.MYSQL_TYPE_DATETIME2: { + /* + * DATETIME and DATE low-level memory and disk representation + * routines 1 bit sign (used when on disk) 17 bits year*13+month + * (year 0-9999, month 0-12) 5 bits day (0-31) 5 bits hour + * (0-23) 6 bits minute (0-59) 6 bits second (0-59) 24 bits + * microseconds (0-999999) Total: 64 bits = 8 bytes + * SYYYYYYY.YYYYYYYY + * .YYdddddh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff + */ + long intpart = buffer.getBeUlong40() - DATETIMEF_INT_OFS; // big-endian + @SuppressWarnings("unused") + int frac = 0; + switch (meta) { + case 0: + frac = 0; + break; + case 1: + case 2: + frac = buffer.getInt8() * 10000; + break; + case 3: + case 4: + frac = buffer.getBeInt16() * 100; + break; + case 5: + case 6: + frac = buffer.getBeInt24(); + break; + default: + frac = 0; + break; + } + + if (intpart == 0) { + value = "0000-00-00 00:00:00"; + } else { + // 构造TimeStamp只处理到秒 + long ymd = intpart >> 17; + long ym = ymd >> 5; + long hms = intpart % (1 << 17); + + // if (cal == null) cal = Calendar.getInstance(); + // cal.clear(); + // cal.set((int) (ym / 13), (int) (ym % 13) - 1, (int) (ymd + // % (1 << 5)), (int) (hms >> 12), + // (int) ((hms >> 6) % (1 << 6)), (int) (hms % (1 << 6))); + // value = new Timestamp(cal.getTimeInMillis()); + value = String.format("%04d-%02d-%02d %02d:%02d:%02d", + (int) (ym / 13), + (int) (ym % 13), + (int) (ymd % (1 << 5)), + (int) (hms >> 12), + (int) ((hms >> 6) % (1 << 6)), + (int) (hms % (1 << 6))); + } + javaType = Types.TIMESTAMP; + length = 5 + (meta + 1) / 2; + break; + } + case LogEvent.MYSQL_TYPE_TIME: { + // MYSQL DataTypes: TIME + // The range is '-838:59:59' to '838:59:59' + // final int i32 = buffer.getUint24(); + final int i32 = buffer.getInt24(); + final int u32 = Math.abs(i32); + if (i32 == 0) { + value = "00:00:00"; + } else { + // if (cal == null) cal = Calendar.getInstance(); + // cal.clear(); + // cal.set(70, 0, 1, i32 / 10000, (i32 % 10000) / 100, i32 % + // 100); + // value = new Time(cal.getTimeInMillis()); + value = String.format("%s%02d:%02d:%02d", + (i32 >= 0) ? "" : "-", + u32 / 10000, + (u32 % 10000) / 100, + u32 % 100); + } + javaType = Types.TIME; + length = 3; + break; + } + case LogEvent.MYSQL_TYPE_TIME2: { + /* + * TIME low-level memory and disk representation routines + * In-memory format: 1 bit sign (Used for sign, when on disk) 1 + * bit unused (Reserved for wider hour range, e.g. for + * intervals) 10 bit hour (0-836) 6 bit minute (0-59) 6 bit + * second (0-59) 24 bits microseconds (0-999999) Total: 48 bits + * = 6 bytes + * Suhhhhhh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff + */ + long intpart = 0; + int frac = 0; + long ltime = 0; + switch (meta) { + case 0: + intpart = buffer.getBeUint24() - TIMEF_INT_OFS; // big-endian + ltime = intpart << 24; + break; + case 1: + case 2: + intpart = buffer.getBeUint24() - TIMEF_INT_OFS; + frac = buffer.getUint8(); + if (intpart < 0 && frac > 0) { + /* + * Negative values are stored with reverse + * fractional part order, for binary sort + * compatibility. Disk value intpart frac Time value + * Memory value 800000.00 0 0 00:00:00.00 + * 0000000000.000000 7FFFFF.FF -1 255 -00:00:00.01 + * FFFFFFFFFF.FFD8F0 7FFFFF.9D -1 99 -00:00:00.99 + * FFFFFFFFFF.F0E4D0 7FFFFF.00 -1 0 -00:00:01.00 + * FFFFFFFFFF.000000 7FFFFE.FF -1 255 -00:00:01.01 + * FFFFFFFFFE.FFD8F0 7FFFFE.F6 -2 246 -00:00:01.10 + * FFFFFFFFFE.FE7960 Formula to convert fractional + * part from disk format (now stored in "frac" + * variable) to absolute value: "0x100 - frac". To + * reconstruct in-memory value, we shift to the next + * integer value and then substruct fractional part. + */ + intpart++; /* Shift to the next integer value */ + frac -= 0x100; /* -(0x100 - frac) */ + // fraclong = frac * 10000; + } + ltime = intpart << 24 + frac * 10000; + break; + case 3: + case 4: + intpart = buffer.getBeUint24() - TIMEF_INT_OFS; + frac = buffer.getBeUint16(); + if (intpart < 0 && frac > 0) { + /* + * Fix reverse fractional part order: + * "0x10000 - frac". See comments for FSP=1 and + * FSP=2 above. + */ + intpart++; /* Shift to the next integer value */ + frac -= 0x10000; /* -(0x10000-frac) */ + // fraclong = frac * 100; + } + ltime = intpart << 24 + frac * 100; + break; + case 5: + case 6: + intpart = buffer.getBeUlong48() - TIMEF_INT_OFS; + ltime = intpart; + break; + default: + intpart = buffer.getBeUint24() - TIMEF_INT_OFS; + ltime = intpart << 24; + break; + } + + if (intpart == 0) { + value = "00:00:00"; + } else { + // 目前只记录秒，不处理us frac + // if (cal == null) cal = Calendar.getInstance(); + // cal.clear(); + // cal.set(70, 0, 1, (int) ((intpart >> 12) % (1 << 10)), + // (int) ((intpart >> 6) % (1 << 6)), + // (int) (intpart % (1 << 6))); + // value = new Time(cal.getTimeInMillis()); + long ultime = Math.abs(ltime); + intpart = ultime >> 24; + value = String.format("%s%02d:%02d:%02d", + ltime >= 0 ? "" : "-", + (int) ((intpart >> 12) % (1 << 10)), + (int) ((intpart >> 6) % (1 << 6)), + (int) (intpart % (1 << 6))); + } + + javaType = Types.TIME; + length = 3 + (meta + 1) / 2; + break; + } + case LogEvent.MYSQL_TYPE_NEWDATE: { + /* + * log_event.h : This enumeration value is only used internally + * and cannot exist in a binlog. + */ + logger.warn("MYSQL_TYPE_NEWDATE : This enumeration value is " + + "only used internally and cannot exist in a binlog!"); + javaType = Types.DATE; + value = null; /* unknown format */ + length = 0; + break; + } + case LogEvent.MYSQL_TYPE_DATE: { + // MYSQL DataTypes: + // range: 0000-00-00 ~ 9999-12-31 + final int i32 = buffer.getUint24(); + if (i32 == 0) { + value = "0000-00-00"; + } else { + // if (cal == null) cal = Calendar.getInstance(); + // cal.clear(); + /* month is 0-based, 0 for january. */ + // cal.set((i32 / (16 * 32)), (i32 / 32 % 16) - 1, (i32 % + // 32)); + // value = new java.sql.Date(cal.getTimeInMillis()); + value = String.format("%04d-%02d-%02d", i32 / (16 * 32), i32 / 32 % 16, i32 % 32); + } + javaType = Types.DATE; + length = 3; + break; + } + case LogEvent.MYSQL_TYPE_YEAR: { + // MYSQL DataTypes: YEAR[(2|4)] + // In four-digit format, values display as 1901 to 2155, and + // 0000. + // In two-digit format, values display as 70 to 69, representing + // years from 1970 to 2069. + + final int i32 = buffer.getUint8(); + // If connection property 'YearIsDateType' has + // set, value is java.sql.Date. + /* + * if (cal == null) cal = Calendar.getInstance(); cal.clear(); + * cal.set(Calendar.YEAR, i32 + 1900); value = new + * java.sql.Date(cal.getTimeInMillis()); + */ + // The else, value is java.lang.Short. + if (i32 == 0) { + value = "0000"; + } else { + value = String.valueOf((short) (i32 + 1900)); + } + // It might seem more correct to create a java.sql.Types.DATE + // value + // for this date, but it is much simpler to pass the value as an + // integer. The MySQL JDBC specification states that one can + // pass a java int between 1901 and 2055. Creating a DATE value + // causes truncation errors with certain SQL_MODES + // (e.g."STRICT_TRANS_TABLES"). + javaType = Types.VARCHAR; // Types.INTEGER; + length = 1; + break; + } + case LogEvent.MYSQL_TYPE_ENUM: { + final int int32; + /* + * log_event.h : This enumeration value is only used internally + * and cannot exist in a binlog. + */ + switch (len) { + case 1: + int32 = buffer.getUint8(); + break; + case 2: + int32 = buffer.getUint16(); + break; + default: + throw new IllegalArgumentException("!! Unknown ENUM packlen = " + len); + } + // logger.warn("MYSQL_TYPE_ENUM : This enumeration value is " + // + "only used internally and cannot exist in a binlog!"); + value = Integer.valueOf(int32); + javaType = Types.INTEGER; + length = len; + break; + } + case LogEvent.MYSQL_TYPE_SET: { + final int nbits = (meta & 0xFF) * 8; + len = (nbits + 7) / 8; + if (nbits > 1) { + // byte[] bits = new byte[len]; + // buffer.fillBytes(bits, 0, len); + // 转化为unsign long + switch (len) { + case 1: + value = buffer.getInt8(); + break; + case 2: + value = buffer.getUint16(); + break; + case 3: + value = buffer.getUint24(); + break; + case 4: + value = buffer.getUint32(); + break; + case 5: + value = buffer.getUlong40(); + break; + case 6: + value = buffer.getUlong48(); + break; + case 7: + value = buffer.getUlong56(); + break; + case 8: + value = buffer.getUlong64(); + break; + default: + throw new IllegalArgumentException("!! Unknown Set len = " + len); + } + } else { + final int bit = buffer.getInt8(); + // value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE; + value = bit; + } + + javaType = Types.BIT; + length = len; + break; + } + case LogEvent.MYSQL_TYPE_TINY_BLOB: { + /* + * log_event.h : This enumeration value is only used internally + * and cannot exist in a binlog. + */ + logger.warn("MYSQL_TYPE_TINY_BLOB : This enumeration value is " + + "only used internally and cannot exist in a binlog!"); + } + case LogEvent.MYSQL_TYPE_MEDIUM_BLOB: { + /* + * log_event.h : This enumeration value is only used internally + * and cannot exist in a binlog. + */ + logger.warn("MYSQL_TYPE_MEDIUM_BLOB : This enumeration value is " + + "only used internally and cannot exist in a binlog!"); + } + case LogEvent.MYSQL_TYPE_LONG_BLOB: { + /* + * log_event.h : This enumeration value is only used internally + * and cannot exist in a binlog. + */ + logger.warn("MYSQL_TYPE_LONG_BLOB : This enumeration value is " + + "only used internally and cannot exist in a binlog!"); + } + case LogEvent.MYSQL_TYPE_BLOB: { + /* + * BLOB or TEXT datatype + */ + switch (meta) { + case 1: { + /* TINYBLOB/TINYTEXT */ + final int len8 = buffer.getUint8(); + byte[] binary = new byte[len8]; + buffer.fillBytes(binary, 0, len8); + value = binary; + javaType = Types.VARBINARY; + length = len8; + break; + } + case 2: { + /* BLOB/TEXT */ + final int len16 = buffer.getUint16(); + byte[] binary = new byte[len16]; + buffer.fillBytes(binary, 0, len16); + value = binary; + javaType = Types.LONGVARBINARY; + length = len16; + break; + } + case 3: { + /* MEDIUMBLOB/MEDIUMTEXT */ + final int len24 = buffer.getUint24(); + byte[] binary = new byte[len24]; + buffer.fillBytes(binary, 0, len24); + value = binary; + javaType = Types.LONGVARBINARY; + length = len24; + break; + } + case 4: { + /* LONGBLOB/LONGTEXT */ + final int len32 = (int) buffer.getUint32(); + byte[] binary = new byte[len32]; + buffer.fillBytes(binary, 0, len32); + value = binary; + javaType = Types.LONGVARBINARY; + length = len32; + break; + } + default: + throw new IllegalArgumentException("!! Unknown BLOB packlen = " + meta); + } + break; + } + case LogEvent.MYSQL_TYPE_VARCHAR: + case LogEvent.MYSQL_TYPE_VAR_STRING: { + /* + * Except for the data length calculation, MYSQL_TYPE_VARCHAR, + * MYSQL_TYPE_VAR_STRING and MYSQL_TYPE_STRING are handled the + * same way. + */ + len = meta; + if (len < 256) { + len = buffer.getUint8(); + } else { + len = buffer.getUint16(); + } + + if (isBinary) { + // fixed issue #66 ,binary类型在binlog中为var_string + /* fill binary */ + byte[] binary = new byte[len]; + buffer.fillBytes(binary, 0, len); + + javaType = Types.VARBINARY; + value = binary; + } else { + value = buffer.getFullString(len, charsetName); + javaType = Types.VARCHAR; + } + + length = len; + break; + } + case LogEvent.MYSQL_TYPE_STRING: { + if (len < 256) { + len = buffer.getUint8(); + } else { + len = buffer.getUint16(); + } + + if (isBinary) { + /* fill binary */ + byte[] binary = new byte[len]; + buffer.fillBytes(binary, 0, len); + + javaType = Types.BINARY; + value = binary; + } else { + value = buffer.getFullString(len, charsetName); + javaType = Types.CHAR; // Types.VARCHAR; + } + length = len; + break; + } + case LogEvent.MYSQL_TYPE_GEOMETRY: { + /* + * MYSQL_TYPE_GEOMETRY: copy from BLOB or TEXT + */ + switch (meta) { + case 1: + len = buffer.getUint8(); + break; + case 2: + len = buffer.getUint16(); + break; + case 3: + len = buffer.getUint24(); + break; + case 4: + len = (int) buffer.getUint32(); + break; + default: + throw new IllegalArgumentException("!! Unknown MYSQL_TYPE_GEOMETRY packlen = " + meta); + } + /* fill binary */ + byte[] binary = new byte[len]; + buffer.fillBytes(binary, 0, len); + + /* Warning unsupport cloumn type */ + logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d", + meta, + meta, + len)); + javaType = Types.BINARY; + value = binary; + length = len; + break; + } + default: + logger.error(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)", + type, + meta, + meta)); + javaType = Types.OTHER; + value = null; + length = 0; + } + + return value; + } + + public final boolean isNull() { + return fNull; + } + + public final int getJavaType() { + return javaType; + } + + public final Serializable getValue() { + return value; + } + + public final int getLength() { + return length; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java new file mode 100644 index 00000000..ca898cac --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java @@ -0,0 +1,221 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.util.BitSet; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogContext; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Common base class for all row-containing log events. + * + * @author Changyuan.lh + * @version 1.0 + */ +public abstract class RowsLogEvent extends LogEvent +{ + /** + * Fixed data part: + * + *
    + *
  • 6 bytes. The table ID.
    • + *
  • 2 bytes. Reserved for future use.
    • + *
+ * + *

+ * Variable data part: + * + *

    + *
  • Packed integer. The number of columns in the table.
    • + * + *
  • Variable-sized. Bit-field indicating whether each column is used, one + * bit per column. For this field, the amount of storage required for N + * columns is INT((N+7)/8) bytes.
    • + * + *
  • Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating + * whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one + * bit per column. For this field, the amount of storage required for N + * columns is INT((N+7)/8) bytes.
    • + * + *
  • Variable-sized. A sequence of zero or more rows. The end is + * determined by the size of the event. Each row has the following format: + * + *
      + *
    • Variable-sized. Bit-field indicating whether each field in the row is + * NULL. Only columns that are "used" according to the second field in the + * variable data part are listed here. If the second field in the variable + * data part has N one-bits, the amount of storage required for this field + * is INT((N+7)/8) bytes.
      • + * + *
    • Variable-sized. The row-image, containing values of all table fields. + * This only lists table fields that are used (according to the second field + * of the variable data part) and non-NULL (according to the previous + * field). In other words, the number of values listed here is equal to the + * number of zero bits in the previous field (not counting padding bits in + * the last byte). The format of each value is described in the + * log_event_print_value() function in log_event.cc.
      • + * + *
    • (for UPDATE_ROWS_EVENT only) the previous two fields are repeated, + * representing a second table row.
      • + *
    + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + private final long tableId; /* Table ID */ + private TableMapLogEvent table; /* The table the rows belong to */ + + /** Bitmap denoting columns available */ + protected final int columnLen; + protected final BitSet columns; + + /** + * Bitmap for columns available in the after image, if present. These fields + * are only available for Update_rows events. Observe that the width of both + * the before image COLS vector and the after image COLS vector is the same: + * the number of columns of the table on the master. + */ + protected final BitSet changeColumns; + + /** XXX: Don't handle buffer in another thread. */ + private final LogBuffer rowsBuf; /* The rows in packed format */ + + /** + * enum enum_flag + * + * These definitions allow you to combine the flags into an appropriate flag + * set using the normal bitwise operators. The implicit conversion from an + * enum-constant to an integer is accepted by the compiler, which is then + * used to set the real set of flags. + */ + private final int flags; + + /** Last event of a statement */ + public static final int STMT_END_F = 1; + + /** Value of the OPTION_NO_FOREIGN_KEY_CHECKS flag in thd->options */ + public static final int NO_FOREIGN_KEY_CHECKS_F = (1 << 1); + + /** Value of the OPTION_RELAXED_UNIQUE_CHECKS flag in thd->options */ + public static final int RELAXED_UNIQUE_CHECKS_F = (1 << 2); + + /** + * Indicates that rows in this event are complete, that is contain values + * for all columns of the table. + */ + public static final int COMPLETE_ROWS_F = (1 << 3); + + /* RW = "RoWs" */ + public static final int RW_MAPID_OFFSET = 0; + public static final int RW_FLAGS_OFFSET = 6; + public static final int RW_VHLEN_OFFSET = 8; + public static final int RW_V_TAG_LEN = 1; + public static final int RW_V_EXTRAINFO_TAG = 0; + + + public RowsLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; + int headerLen = 0; + buffer.position(commonHeaderLen + RW_MAPID_OFFSET); + if (postHeaderLen == 6) + { + /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */ + tableId = buffer.getUint32(); + } + else + { + tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET + } + flags = buffer.getUint16(); + + if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2) + { + headerLen = buffer.getUint16(); + headerLen -= 2; + int start = buffer.position(); + int end = start + headerLen; + for(int i = start ;i < end; ){ + switch (buffer.getUint8(i++)) { + case RW_V_EXTRAINFO_TAG: + // int infoLen = buffer.getUint8(); + buffer.position(i + EXTRA_ROW_INFO_LEN_OFFSET); + int checkLen = buffer.getUint8(); // EXTRA_ROW_INFO_LEN_OFFSET + int val= checkLen - EXTRA_ROW_INFO_HDR_BYTES; + assert(buffer.getUint8() == val); //EXTRA_ROW_INFO_FORMAT_OFFSET + for (int j= 0; j < val; j++) { + assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i + } + break; + default: + i = end; + break; + } + } + } + + buffer.position(commonHeaderLen + postHeaderLen + headerLen); + columnLen = (int) buffer.getPackedLong(); + columns = buffer.getBitmap(columnLen); + + if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT) + { + changeColumns = buffer.getBitmap(columnLen); + } + else + { + changeColumns = columns; + } + + // XXX: Don't handle buffer in another thread. + int dataSize = buffer.limit() - buffer.position(); + rowsBuf = buffer.duplicate(dataSize); + } + + public final void fillTable(LogContext context) + { + table = context.getTable(tableId); + + // end of statement check: + if ((flags & RowsLogEvent.STMT_END_F) != 0) + { + // Now is safe to clear ignored map (clear_tables will also + // delete original table map events stored in the map). + context.clearAllTables(); + } + } + + public final long getTableId() + { + return tableId; + } + + public final TableMapLogEvent getTable() + { + return table; + } + + public final BitSet getColumns() + { + return columns; + } + + public final BitSet getChangeColumns() + { + return changeColumns; + } + + public final RowsLogBuffer getRowsBuf(String charsetName) + { + return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName); + } + + public final int getFlags(final int flags) + { + return this.flags & flags; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java new file mode 100644 index 00000000..16293ae1 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java @@ -0,0 +1,33 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * @author jianghang 2013-4-8 上午12:36:29 + * @version 1.0.3 + * @since mysql 5.6 + */ +public class RowsQueryLogEvent extends IgnorableLogEvent { + + private String rowsQuery; + + public RowsQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header, buffer, descriptionEvent); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; + + /* + * m_rows_query length is stored using only one byte, but that length is + * ignored and the complete query is read. + */ + int offset = commonHeaderLen + postHeaderLen + 1; + int len = buffer.limit() - offset; + rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1); + } + + public String getRowsQuery() { + return rowsQuery; + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java new file mode 100644 index 00000000..29054055 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java @@ -0,0 +1,61 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and + * 4.x). + * + * Format_description_log_event derives from Start_log_event_v3; it is the + * Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that + * describes the other events' Common-Header/Post-Header lengths. This event is + * sent by MySQL 5.0 whenever it starts sending a new binlog if the requested + * position is >4 (otherwise if ==4 the event will be sent naturally). + * + * @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3 + * + * @author Changyuan.lh + * @version 1.0 + */ +public class StartLogEventV3 extends LogEvent +{ + /** + * We could have used SERVER_VERSION_LENGTH, but this introduces an obscure + * dependency - if somebody decided to change SERVER_VERSION_LENGTH this + * would break the replication protocol + */ + public static final int ST_SERVER_VER_LEN = 50; + + /* start event post-header (for v3 and v4) */ + public static final int ST_BINLOG_VER_OFFSET = 0; + public static final int ST_SERVER_VER_OFFSET = 2; + + protected int binlogVersion; + protected String serverVersion; + + public StartLogEventV3(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + buffer.position(descriptionEvent.commonHeaderLen); + binlogVersion = buffer.getUint16(); // ST_BINLOG_VER_OFFSET + serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET + } + + public StartLogEventV3() + { + super(new LogHeader(START_EVENT_V3)); + } + + public final String getServerVersion() + { + return serverVersion; + } + + public final int getBinlogVersion() + { + return binlogVersion; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java new file mode 100644 index 00000000..49147629 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java @@ -0,0 +1,22 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Stop_log_event. + * + * The Post-Header and Body for this event type are empty; it only has the + * Common-Header. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class StopLogEvent extends LogEvent +{ + public StopLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent description_event) + { + super(header); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java new file mode 100644 index 00000000..3501d30e --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java @@ -0,0 +1,543 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.util.BitSet; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * In row-based mode, every row operation event is preceded by a + * Table_map_log_event which maps a table definition to a number. The table + * definition consists of database name, table name, and column definitions. + * + * The Post-Header has the following components: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Post-Header for Table_map_log_event
NameFormatDescription
table_id6 bytes unsigned integerThe number that identifies the table.
flags2 byte bitfieldReserved for future use; currently always 0.
+ * + * The Body has the following components: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Body for Table_map_log_event
NameFormatDescription
database_nameone byte string length, followed by null-terminated stringThe name of the database in which the table resides. The name is + * represented as a one byte unsigned integer representing the number of bytes + * in the name, followed by length bytes containing the database name, followed + * by a terminating 0 byte. (Note the redundancy in the representation of the + * length.)
table_nameone byte string length, followed by null-terminated stringThe name of the table, encoded the same way as the database name above.
column_countpacked_integer "Packed Integer"The number of columns in the table, represented as a packed + * variable-length integer.
column_typeList of column_count 1 byte enumeration valuesThe type of each column in the table, listed from left to right. Each + * byte is mapped to a column type according to the enumeration type + * enum_field_types defined in mysql_com.h. The mapping of types to numbers is + * listed in the table Table_table_map_log_event_column_types "below" (along + * with description of the associated metadata field).
metadata_lengthpacked_integer "Packed Integer"The length of the following metadata block
metadatalist of metadata for each columnFor each column from left to right, a chunk of data who's length and + * semantics depends on the type of the column. The length and semantics for the + * metadata for each column are listed in the table + * Table_table_map_log_event_column_types "below".
null_bitscolumn_count bits, rounded up to nearest byteFor each column, a bit indicating whether data in the column can be NULL + * or not. The number of bytes needed for this is int((column_count+7)/8). The + * flag for the first column from the left is in the least-significant bit of + * the first byte, the second is in the second least significant bit of the + * first byte, the ninth is in the least significant bit of the second byte, and + * so on.
+ * + * The table below lists all column types, along with the numerical identifier + * for it and the size and interpretation of meta-data used to describe the + * type. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Table_map_log_event column types: numerical identifier and + * metadata
NameIdentifierSize of metadata in bytesDescription of metadata
MYSQL_TYPE_DECIMAL00No column metadata.
MYSQL_TYPE_TINY10No column metadata.
MYSQL_TYPE_SHORT20No column metadata.
MYSQL_TYPE_LONG30No column metadata.
MYSQL_TYPE_FLOAT41 byte1 byte unsigned integer, representing the "pack_length", which is equal + * to sizeof(float) on the server from which the event originates.
MYSQL_TYPE_DOUBLE51 byte1 byte unsigned integer, representing the "pack_length", which is equal + * to sizeof(double) on the server from which the event originates.
MYSQL_TYPE_NULL60No column metadata.
MYSQL_TYPE_TIMESTAMP70No column metadata.
MYSQL_TYPE_LONGLONG80No column metadata.
MYSQL_TYPE_INT2490No column metadata.
MYSQL_TYPE_DATE100No column metadata.
MYSQL_TYPE_TIME110No column metadata.
MYSQL_TYPE_DATETIME120No column metadata.
MYSQL_TYPE_YEAR130No column metadata.
MYSQL_TYPE_NEWDATE14–This enumeration value is only used internally and cannot exist in a + * binlog.
MYSQL_TYPE_VARCHAR152 bytes2 byte unsigned integer representing the maximum length of the string.
MYSQL_TYPE_BIT162 bytesA 1 byte unsigned int representing the length in bits of the bitfield (0 + * to 64), followed by a 1 byte unsigned int representing the number of bytes + * occupied by the bitfield. The number of bytes is either int((length+7)/8) or + * int(length/8).
MYSQL_TYPE_NEWDECIMAL2462 bytesA 1 byte unsigned int representing the precision, followed by a 1 byte + * unsigned int representing the number of decimals.
MYSQL_TYPE_ENUM247–This enumeration value is only used internally and cannot exist in a + * binlog.
MYSQL_TYPE_SET248–This enumeration value is only used internally and cannot exist in a + * binlog.
MYSQL_TYPE_TINY_BLOB249–This enumeration value is only used internally and cannot exist in a + * binlog.
MYSQL_TYPE_MEDIUM_BLOB250–This enumeration value is only used internally and cannot exist in a + * binlog.
MYSQL_TYPE_LONG_BLOB251–This enumeration value is only used internally and cannot exist in a + * binlog.
MYSQL_TYPE_BLOB2521 byteThe pack length, i.e., the number of bytes needed to represent the length + * of the blob: 1, 2, 3, or 4.
MYSQL_TYPE_VAR_STRING2532 bytesThis is used to store both strings and enumeration values. The first byte + * is a enumeration value storing the real type, which may be either + * MYSQL_TYPE_VAR_STRING or MYSQL_TYPE_ENUM. The second byte is a 1 byte + * unsigned integer representing the field size, i.e., the number of bytes + * needed to store the length of the string.
MYSQL_TYPE_STRING2542 bytesThe first byte is always MYSQL_TYPE_VAR_STRING (i.e., 253). The second + * byte is the field size, i.e., the number of bytes in the representation of + * size of the string: 3 or 4.
MYSQL_TYPE_GEOMETRY2551 byteThe pack length, i.e., the number of bytes needed to represent the length + * of the geometry: 1, 2, 3, or 4.
+ * + * @author Changyuan.lh + * @version 1.0 + */ +public final class TableMapLogEvent extends LogEvent +{ + /** + * Fixed data part: + *
    + *
  • 6 bytes. The table ID.
    • + *
  • 2 bytes. Reserved for future use.
    • + *
+ * + *

+ * Variable data part: + *

    + *
  • 1 byte. The length of the database name.
    • + *
  • Variable-sized. The database name (null-terminated).
    • + *
  • 1 byte. The length of the table name.
    • + *
  • Variable-sized. The table name (null-terminated).
    • + *
  • Packed integer. The number of columns in the table.
    • + *
  • Variable-sized. An array of column types, one byte per column.
    • + *
  • Packed integer. The length of the metadata block.
    • + *
  • Variable-sized. The metadata block; see log_event.h for contents and + * format.
    • + *
  • Variable-sized. Bit-field indicating whether each column can be NULL, + * one bit per column. For this field, the amount of storage required for N + * columns is INT((N+7)/8) bytes.
    • + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + protected final String dbname; + protected final String tblname; + + /** + * Holding mysql column information. + */ + public static final class ColumnInfo + { + public int type; + public int meta; + } + + protected final int columnCnt; + protected final ColumnInfo[] columnInfo; // buffer for field metadata + + protected final long tableId; + protected BitSet nullBits; + + /** TM = "Table Map" */ + public static final int TM_MAPID_OFFSET = 0; + public static final int TM_FLAGS_OFFSET = 6; + + /** + * Constructor used by slave to read the event from the binary log. + */ + public TableMapLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + final int commonHeaderLen = descriptionEvent.commonHeaderLen; + final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1]; + /* Read the post-header */ + buffer.position(commonHeaderLen + TM_MAPID_OFFSET); + if (postHeaderLen == 6) + { + /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */ + tableId = buffer.getUint32(); + } + else + { + // DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN); + tableId = buffer.getUlong48(); + } + // flags = buffer.getUint16(); + + /* Read the variable part of the event */ + buffer.position(commonHeaderLen + postHeaderLen); + dbname = buffer.getString(); + buffer.forward(1); /* termination null */ + tblname = buffer.getString(); + buffer.forward(1); /* termination null */ + + // Read column information from buffer + columnCnt = (int) buffer.getPackedLong(); + columnInfo = new ColumnInfo[columnCnt]; + for (int i = 0; i < columnCnt; i++) + { + ColumnInfo info = new ColumnInfo(); + info.type = buffer.getUint8(); + columnInfo[i] = info; + } + + if (buffer.position() < buffer.limit()) + { + final int fieldSize = (int) buffer.getPackedLong(); + decodeFields(buffer, fieldSize); + nullBits = buffer.getBitmap(columnCnt); + } + } + + /** + * Decode field metadata by column types. + * + * @see mysql-5.1.60/sql/rpl_utility.h + */ + private final void decodeFields(LogBuffer buffer, final int len) + { + final int limit = buffer.limit(); + + buffer.limit(len + buffer.position()); + for (int i = 0; i < columnCnt; i++) + { + ColumnInfo info = columnInfo[i]; + + switch (info.type) + { + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_DOUBLE: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_GEOMETRY: + /* + These types store a single byte. + */ + info.meta = buffer.getUint8(); + break; + case MYSQL_TYPE_SET: + case MYSQL_TYPE_ENUM: + /* + * log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This + * enumeration value is only used internally and cannot + * exist in a binlog. + */ + logger.warn("This enumeration value is only used internally " + + "and cannot exist in a binlog: type=" + info.type); + break; + case MYSQL_TYPE_STRING: + { + /* + * log_event.h : The first byte is always + * MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the + * field size, i.e., the number of bytes in the + * representation of size of the string: 3 or 4. + */ + int x = (buffer.getUint8() << 8); // real_type + x += buffer.getUint8(); // pack or field length + info.meta = x; + break; + } + case MYSQL_TYPE_BIT: + info.meta = buffer.getUint16(); + break; + case MYSQL_TYPE_VARCHAR: + /* + * These types store two bytes. + */ + info.meta = buffer.getUint16(); + break; + case MYSQL_TYPE_NEWDECIMAL: + { + int x = buffer.getUint8() << 8; // precision + x += buffer.getUint8(); // decimals + info.meta = x; + break; + } + case MYSQL_TYPE_TIME2: + case MYSQL_TYPE_DATETIME2: + case MYSQL_TYPE_TIMESTAMP2: + { + info.meta = buffer.getUint8(); + break; + } + default: + info.meta = 0; + break; + } + } + buffer.limit(limit); + } + + public final String getDbName() + { + return dbname; + } + + public final String getTableName() + { + return tblname; + } + + public final int getColumnCnt() + { + return columnCnt; + } + + public final ColumnInfo[] getColumnInfo() + { + return columnInfo; + } + + public final long getTableId() + { + return tableId; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java new file mode 100644 index 00000000..30dedd8a --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java @@ -0,0 +1,17 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Unknown_log_event + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class UnknownLogEvent extends LogEvent +{ + public UnknownLogEvent(LogHeader header) + { + super(header); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java new file mode 100644 index 00000000..ed946f3a --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java @@ -0,0 +1,22 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * Log row updates with a before image. The event contain several update rows + * for a table. Note that each event contains only rows for one table. + * + * Also note that the row data consists of pairs of row data: one row for the + * old data and one row for the new data. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class UpdateRowsLogEvent extends RowsLogEvent +{ + public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header, buffer, descriptionEvent); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java new file mode 100644 index 00000000..ffd2c4eb --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java @@ -0,0 +1,146 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import java.io.IOException; +import java.io.Serializable; + +import com.taobao.tddl.dbsync.binlog.CharsetConversion; +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * User_var_log_event. + * + * Every time a query uses the value of a user variable, a User_var_log_event is + * written before the Query_log_event, to set the user variable. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class UserVarLogEvent extends LogEvent +{ + /** + * Fixed data part: Empty + * + *

+ * Variable data part: + * + *

    + *
  • 4 bytes. the size of the user variable name.
    • + *
  • The user variable name.
    • + *
  • 1 byte. Non-zero if the variable value is the SQL NULL value, 0 + * otherwise. If this byte is 0, the following parts exist in the event.
    • + *
  • 1 byte. The user variable type. The value corresponds to elements of + * enum Item_result defined in include/mysql_com.h.
    • + *
  • 4 bytes. The number of the character set for the user variable + * (needed for a string variable). The character set number is really a + * collation number that indicates a character set/collation pair.
    • + *
  • 4 bytes. The size of the user variable value (corresponds to member + * val_len of class Item_string).
    • + *
  • Variable-sized. For a string variable, this is the string. For a + * float or integer variable, this is its value in 8 bytes.
    • + *
+ * + * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log + */ + private final String name; + private final Serializable value; + private final int type; + private final int charsetNumber; + private final boolean isNull; + + /** + * The following is for user defined functions + * + * @see mysql-5.1.60//include/mysql_com.h + */ + public static final int STRING_RESULT = 0; + public static final int REAL_RESULT = 1; + public static final int INT_RESULT = 2; + public static final int ROW_RESULT = 3; + public static final int DECIMAL_RESULT = 4; + + /* User_var event data */ + public static final int UV_VAL_LEN_SIZE = 4; + public static final int UV_VAL_IS_NULL = 1; + public static final int UV_VAL_TYPE_SIZE = 1; + public static final int UV_NAME_LEN_SIZE = 4; + public static final int UV_CHARSET_NUMBER_SIZE = 4; + + public UserVarLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) throws IOException + { + super(header); + + /* The Post-Header is empty. The Variable Data part begins immediately. */ + buffer.position(descriptionEvent.commonHeaderLen + + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]); + final int nameLen = (int) buffer.getUint32(); + name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE + isNull = (0 != buffer.getInt8()); + + if (isNull) + { + type = STRING_RESULT; + charsetNumber = 63; /* binary */ + value = null; + } + else + { + type = buffer.getInt8(); // UV_VAL_IS_NULL + charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE + final int valueLen = (int) buffer.getUint32(); // buf + UV_CHARSET_NUMBER_SIZE + final int limit = buffer.limit(); /* for restore */ + buffer.limit(buffer.position() + valueLen); + + /* @see User_var_log_event::print */ + switch (type) + { + case REAL_RESULT: + value = Double.valueOf(buffer.getDouble64()); // float8get + break; + case INT_RESULT: + if (valueLen == 8) + value = Long.valueOf(buffer.getLong64()); // !uint8korr + else if (valueLen == 4) + value = Long.valueOf(buffer.getUint32()); + else + throw new IOException("Error INT_RESULT length: " + + valueLen); + break; + case DECIMAL_RESULT: + final int precision = buffer.getInt8(); + final int scale = buffer.getInt8(); + value = buffer.getDecimal(precision, scale); // bin2decimal + break; + case STRING_RESULT: + String charsetName = CharsetConversion.getJavaCharset(charsetNumber); + value = buffer.getFixString(valueLen, charsetName); + break; + case ROW_RESULT: + // this seems to be banned in MySQL altogether + throw new IOException("ROW_RESULT is unsupported"); + default: + value = null; + break; + } + buffer.limit(limit); + } + } + + public final String getQuery() + { + if (value == null) + { + return "SET @" + name + " := NULL"; + } + else if (type == STRING_RESULT) + { + // TODO: do escaping !? + return "SET @" + name + " := \'" + value + '\''; + } + else + { + return "SET @" + name + " := " + String.valueOf(value); + } + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java new file mode 100644 index 00000000..c0f68b23 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java @@ -0,0 +1,19 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +/** + * Log row insertions and updates. The event contain several insert/update rows + * for a table. Note that each event contains only rows for one table. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class WriteRowsLogEvent extends RowsLogEvent +{ + public WriteRowsLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header, buffer, descriptionEvent); + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java new file mode 100644 index 00000000..1737f4ab --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java @@ -0,0 +1,32 @@ +package com.taobao.tddl.dbsync.binlog.event; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.LogEvent; + +/** + * Logs xid of the transaction-to-be-committed in the 2pc protocol. Has no + * meaning in replication, slaves ignore it. + * + * @author Changyuan.lh + * @version 1.0 + */ +public final class XidLogEvent extends LogEvent +{ + private final long xid; + + public XidLogEvent(LogHeader header, LogBuffer buffer, + FormatDescriptionLogEvent descriptionEvent) + { + super(header); + + /* The Post-Header is empty. The Variable Data part begins immediately. */ + buffer.position(descriptionEvent.commonHeaderLen + + descriptionEvent.postHeaderLen[XID_EVENT - 1]); + xid = buffer.getLong64(); // !uint8korr + } + + public final long getXid() + { + return xid; + } +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java new file mode 100644 index 00000000..cbb83871 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java @@ -0,0 +1,33 @@ +package com.taobao.tddl.dbsync.binlog.event.mariadb; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; +import com.taobao.tddl.dbsync.binlog.event.LogHeader; + +/** + * mariadb的ANNOTATE_ROWS_EVENT类型 + * + * @author jianghang 2014-1-20 下午2:20:35 + * @since 1.0.17 + */ +public class AnnotateRowsEvent extends IgnorableLogEvent { + + private String rowsQuery; + + public AnnotateRowsEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header, buffer, descriptionEvent); + + final int commonHeaderLen = descriptionEvent.getCommonHeaderLen(); + final int postHeaderLen = descriptionEvent.getPostHeaderLen()[header.getType() - 1]; + + int offset = commonHeaderLen + postHeaderLen; + int len = buffer.limit() - offset; + rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1); + } + + public String getRowsQuery() { + return rowsQuery; + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java new file mode 100644 index 00000000..aaa2a286 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java @@ -0,0 +1,21 @@ +package com.taobao.tddl.dbsync.binlog.event.mariadb; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; +import com.taobao.tddl.dbsync.binlog.event.LogHeader; + +/** + * mariadb10的BINLOG_CHECKPOINT_EVENT类型 + * + * @author jianghang 2014-1-20 下午2:22:04 + * @since 1.0.17 + */ +public class BinlogCheckPointLogEvent extends IgnorableLogEvent { + + public BinlogCheckPointLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header, buffer, descriptionEvent); + // do nothing , just mariadb binlog checkpoint + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java new file mode 100644 index 00000000..a2b7be95 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java @@ -0,0 +1,21 @@ +package com.taobao.tddl.dbsync.binlog.event.mariadb; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; +import com.taobao.tddl.dbsync.binlog.event.LogHeader; + +/** + * mariadb的GTID_LIST_EVENT类型 + * + * @author jianghang 2014-1-20 下午4:51:50 + * @since 1.0.17 + */ +public class MariaGtidListLogEvent extends IgnorableLogEvent { + + public MariaGtidListLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header, buffer, descriptionEvent); + // do nothing , just ignore log event + } + +} diff --git a/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java new file mode 100644 index 00000000..9dc22b39 --- /dev/null +++ b/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java @@ -0,0 +1,21 @@ +package com.taobao.tddl.dbsync.binlog.event.mariadb; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; +import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent; +import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent; +import com.taobao.tddl.dbsync.binlog.event.LogHeader; + +/** + * mariadb的GTID_EVENT类型 + * + * @author jianghang 2014-1-20 下午4:49:10 + * @since 1.0.17 + */ +public class MariaGtidLogEvent extends IgnorableLogEvent { + + public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){ + super(header, buffer, descriptionEvent); + // do nothing , just ignore log event + } + +} diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java new file mode 100644 index 00000000..65f9ce69 --- /dev/null +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java @@ -0,0 +1,109 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.BitSet; + +import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer; +import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent; +import com.taobao.tddl.dbsync.binlog.event.XidLogEvent; +import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo; +import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; + +public class BaseLogFetcherTest { + + protected String binlogFileName = "mysql-bin.000001"; + protected Charset charset = Charset.forName("utf-8"); + + protected void parseQueryEvent(QueryLogEvent event) { + System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen(), + event.getCatalog())); + System.out.println("sql : " + event.getQuery()); + } + + protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception { + System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen())); + System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name())); + } + + protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception { + System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen())); + System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name())); + } + + protected void parseXidEvent(XidLogEvent event) throws Exception { + System.out.println(String.format("================> binlog[%s:%s]", binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen())); + System.out.println("xid : " + event.getXid()); + } + + + protected void parseRowsEvent(RowsLogEvent event) { + try { + System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName, + event.getHeader().getLogPos() - event.getHeader().getEventLen(), + event.getTable().getDbName(), event.getTable().getTableName())); + RowsLogBuffer buffer = event.getRowsBuf(charset.name()); + BitSet columns = event.getColumns(); + BitSet changeColumns = event.getChangeColumns(); + while (buffer.nextOneRow(columns)) { + // 处理row记录 + int type = event.getHeader().getType(); + if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) { + // insert的记录放在before字段中 + parseOneRow(event, buffer, columns, true); + } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) { + // delete的记录放在before字段中 + parseOneRow(event, buffer, columns, false); + } else { + // update需要处理before/after + System.out.println("-------> before"); + parseOneRow(event, buffer, columns, false); + if (!buffer.nextOneRow(changeColumns)) { + break; + } + System.out.println("-------> after"); + parseOneRow(event, buffer, changeColumns, true); + } + + } + } catch (Exception e) { + throw new RuntimeException("parse row data failed.", e); + } + } + + protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter) + throws UnsupportedEncodingException { + TableMapLogEvent map = event.getTable(); + if (map == null) { + throw new RuntimeException("not found TableMap with tid=" + event.getTableId()); + } + + final int columnCnt = map.getColumnCnt(); + final ColumnInfo[] columnInfo = map.getColumnInfo(); + + for (int i = 0; i < columnCnt; i++) { + ColumnInfo info = columnInfo[i]; + buffer.nextValue(info.type, info.meta); + + if (buffer.isNull()) { + // + } else { + final Serializable value = buffer.getValue(); + if (value instanceof byte[]) { + System.out.println(new String((byte[]) value)); + } else { + System.out.println(value); + } + } + } + + } +} diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java new file mode 100644 index 00000000..375c6643 --- /dev/null +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java @@ -0,0 +1,90 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + +import junit.framework.Assert; + +import org.junit.Test; + +import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.XidLogEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; + +public class DirectLogFetcherTest extends BaseLogFetcherTest { + + @Test + public void testSimple() { + DirectLogFetcher fecther = new DirectLogFetcher(); + try { + Class.forName("com.mysql.jdbc.Driver"); + Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "xxxxx", "xxxxx"); + Statement statement = connection.createStatement(); + statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'"); + statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'"); + + fecther.open(connection, "mysql-bin.000003", 4L, 2); + + LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); + LogContext context = new LogContext(); + while (fecther.fetch()) { + LogEvent event = null; + event = decoder.decode(fecther, context); + + if (event == null) { + throw new RuntimeException("parse failed"); + } + + int eventType = event.getHeader().getType(); + switch (eventType) { + case LogEvent.ROTATE_EVENT: + binlogFileName = ((RotateLogEvent) event).getFilename(); + break; + case LogEvent.WRITE_ROWS_EVENT_V1: + case LogEvent.WRITE_ROWS_EVENT: + parseRowsEvent((WriteRowsLogEvent) event); + break; + case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.UPDATE_ROWS_EVENT: + parseRowsEvent((UpdateRowsLogEvent) event); + break; + case LogEvent.DELETE_ROWS_EVENT_V1: + case LogEvent.DELETE_ROWS_EVENT: + parseRowsEvent((DeleteRowsLogEvent) event); + break; + case LogEvent.QUERY_EVENT: + parseQueryEvent((QueryLogEvent) event); + break; + case LogEvent.ROWS_QUERY_LOG_EVENT: + parseRowsQueryEvent((RowsQueryLogEvent) event); + break; + case LogEvent.ANNOTATE_ROWS_EVENT: + parseAnnotateRowsEvent((AnnotateRowsEvent) event); + break; + case LogEvent.XID_EVENT: + parseXidEvent((XidLogEvent) event); + break; + default: + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + try { + fecther.close(); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + + } +} diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java new file mode 100644 index 00000000..017214d8 --- /dev/null +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java @@ -0,0 +1,93 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.io.File; +import java.io.IOException; +import java.net.URL; + +import junit.framework.Assert; + +import org.junit.Before; +import org.junit.Test; + +import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent; +import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent; +import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent; +import com.taobao.tddl.dbsync.binlog.event.XidLogEvent; +import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent; + +public class FileLogFetcherTest extends BaseLogFetcherTest { + + private String directory; + + @Before + public void setUp() { + URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt"); + File dummyFile = new File(url.getFile()); + directory = new File(dummyFile.getParent() + "/binlog").getPath(); + // directory = "/home/jianghang/tmp/binlog"; + } + + @Test + public void testSimple() { + FileLogFetcher fetcher = new FileLogFetcher(1024 * 16); + try { + LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT); + LogContext context = new LogContext(); + + File current = new File(directory, "mysql-bin.000006"); + fetcher.open(current); + context.setLogPosition(new LogPosition(current.getName())); + + while (fetcher.fetch()) { + LogEvent event = null; + event = decoder.decode(fetcher, context); + if (event != null) { + int eventType = event.getHeader().getType(); + switch (eventType) { + case LogEvent.ROTATE_EVENT: + binlogFileName = ((RotateLogEvent) + event).getFilename(); + break; + case LogEvent.WRITE_ROWS_EVENT_V1: + case LogEvent.WRITE_ROWS_EVENT: + parseRowsEvent((WriteRowsLogEvent) event); + break; + case LogEvent.UPDATE_ROWS_EVENT_V1: + case LogEvent.UPDATE_ROWS_EVENT: + parseRowsEvent((UpdateRowsLogEvent) event); + break; + case LogEvent.DELETE_ROWS_EVENT_V1: + case LogEvent.DELETE_ROWS_EVENT: + parseRowsEvent((DeleteRowsLogEvent) event); + break; + case LogEvent.QUERY_EVENT: + parseQueryEvent((QueryLogEvent) event); + break; + case LogEvent.ROWS_QUERY_LOG_EVENT: + parseRowsQueryEvent((RowsQueryLogEvent) event); + break; + case LogEvent.ANNOTATE_ROWS_EVENT: + parseAnnotateRowsEvent((AnnotateRowsEvent) event); + break; + case LogEvent.XID_EVENT: + parseXidEvent((XidLogEvent) event); + break; + default: + break; + } + } + } + } catch (Exception e) { + Assert.fail(e.getMessage()); + } finally { + try { + fetcher.close(); + } catch (IOException e) { + Assert.fail(e.getMessage()); + } + } + } +} diff --git a/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java new file mode 100644 index 00000000..e551460c --- /dev/null +++ b/dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java @@ -0,0 +1,356 @@ +package com.taobao.tddl.dbsync.binlog; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import junit.framework.TestCase; + +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +public class LogBufferTest extends TestCase +{ + public static final int LOOP = 10000; + + public void testSigned() + { + byte[] array = { 0, 0, 0, (byte) 0xff }; + + LogBuffer buffer = new LogBuffer(array, 0, array.length); + + System.out.println(buffer.getInt32(0)); + System.out.println(buffer.getUint32(0)); + + System.out.println(buffer.getInt24(1)); + System.out.println(buffer.getUint24(1)); + } + + public void testBigInteger() + { + byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff }; + + LogBuffer buffer = new LogBuffer(array, 0, array.length); + + long tt1 = 0; + long l1 = 0; + for (int i = 0; i < LOOP; i++) + { + final long t1 = System.nanoTime(); + l1 = buffer.getLong64(0); + tt1 += System.nanoTime() - t1; + } + System.out.print(tt1 / LOOP); + System.out.print("ns >> "); + System.out.println(l1); + + long tt2 = 0; + BigInteger l2 = null; + for (int i = 0; i < LOOP; i++) + { + final long t2 = System.nanoTime(); + l2 = buffer.getUlong64(0); + tt2 += System.nanoTime() - t2; + } + System.out.print(tt2 / LOOP); + System.out.print("ns >> "); + System.out.println(l2); + } + + /* Reads big-endian integer from no more than 4 bytes */ + private static int convertNBytesToInt(byte[] buffer, int offset, int length) + { + int ret = 0; + for (int i = offset; i < (offset + length); i++) + { + ret = (ret << 8) | (0xff & buffer[i]); + } + return ret; + } + + private static int convert4BytesToInt(byte[] buffer, int offset) + { + int value; + value = (0xff & buffer[offset + 3]); + value += (0xff & buffer[offset + 2]) << 8; + value += (0xff & buffer[offset + 1]) << 16; + value += (0xff & buffer[offset]) << 24; + return value; + } + + public static short convert1ByteToShort(byte[] buffer, int offset) + { + short value; + value = (short) buffer[offset + 0]; + return value; + } + + public static short convert2bytesToShort(byte[] buffer, int offset) + { + short value; + value = (short) (buffer[offset + 0] << 8); + value += (short) (buffer[offset + 1] & 0xff); + return value; + } + + public static final BigDecimal extractDecimal(byte[] buffer, int precision, + int scale) + { + // + // Decimal representation in binlog seems to be as follows: + // 1 byte - 'precision' + // 1 byte - 'scale' + // remaining n bytes - integer such that value = n / (10^scale) + // Integer is represented as follows: + // 1st bit - sign such that set == +, unset == - + // every 4 bytes represent 9 digits in big-endian order, so that if + // you print the values of these quads as big-endian integers one after + // another, you get the whole number string representation in decimal. + // What remains is to put a sign and a decimal dot. + // 13 0a 80 00 00 05 1b 38 b0 60 00 means: + // 0x13 - precision = 19 + // 0x0a - scale = 10 + // 0x80 - positive + // 0x00000005 0x1b38b060 0x00 + // 5 456700000 0 + // 54567000000 / 10^{10} = 5.4567 + // + // int_size below shows how long is integer part + // + // offset = offset + 2; // offset of the number part + // + int intg = precision - scale; + int intg0 = intg / LogBuffer.DIG_PER_INT32; + int frac0 = scale / LogBuffer.DIG_PER_INT32; + int intg0x = intg - intg0 * LogBuffer.DIG_PER_INT32; + int frac0x = scale - frac0 * LogBuffer.DIG_PER_INT32; + + int offset = 0; + + int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1; + + // how many bytes are used to represent given amount of digits? + int integerSize = intg0 * LogBuffer.SIZE_OF_INT32 + + LogBuffer.dig2bytes[intg0x]; + int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32 + + LogBuffer.dig2bytes[frac0x]; + + int bin_size = integerSize + decimalSize; // total bytes + byte[] d_copy = new byte[bin_size]; + + if (bin_size > buffer.length) + { + throw new ArrayIndexOutOfBoundsException("Calculated bin_size: " + + bin_size + ", available bytes: " + buffer.length); + } + + // Invert first bit + d_copy[0] = buffer[0]; + d_copy[0] ^= 0x80; + if (sign == -1) + { + // Invert every byte + d_copy[0] ^= 0xFF; + } + + for (int i = 1; i < bin_size; i++) + { + d_copy[i] = buffer[i]; + if (sign == -1) + { + // Invert every byte + d_copy[i] ^= 0xFF; + } + } + + // Integer part + offset = LogBuffer.dig2bytes[intg0x]; + + BigDecimal intPart = new BigDecimal(0); + + if (offset > 0) + intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset)); + + while (offset < integerSize) + { + intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add( + BigDecimal.valueOf(convert4BytesToInt(d_copy, offset))); + offset += 4; + } + + // Decimal part + BigDecimal fracPart = new BigDecimal(0); + int shift = 0; + for (int i = 0; i < frac0; i++) + { + shift += LogBuffer.DIG_PER_DEC1; + fracPart = fracPart.add(BigDecimal.valueOf( + convert4BytesToInt(d_copy, offset)).movePointLeft(shift)); + offset += 4; + } + + if (LogBuffer.dig2bytes[frac0x] > 0) + { + fracPart = fracPart.add(BigDecimal.valueOf( + convertNBytesToInt(d_copy, offset, + LogBuffer.dig2bytes[frac0x])).movePointLeft( + shift + frac0x)); + } + + return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart)); + } + + public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b, + 0x38, (byte) 0xb0, 0x60, 0x00 }; + + public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff, + (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f, + (byte) 0xa0, (byte) 0xff }; + + public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 }; + + public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 }; + + public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96, + -108 }; + + public void testBigDecimal() throws InterruptedException + { + do + { + System.out.println("old extract decimal: "); + + long tt1 = 0; + BigDecimal bd1 = null; + for (int i = 0; i < LOOP; i++) + { + final long t1 = System.nanoTime(); + bd1 = extractDecimal(array2, 19, 10); + tt1 += System.nanoTime() - t1; + } + System.out.print(tt1 / LOOP); + System.out.print("ns >> "); + System.out.println(bd1); + + long tt2 = 0; + BigDecimal bd2 = null; + for (int i = 0; i < LOOP; i++) + { + final long t2 = System.nanoTime(); + bd2 = extractDecimal(array1, 19, 10); + tt2 += System.nanoTime() - t2; + } + System.out.print(tt2 / LOOP); + System.out.print("ns >> "); + System.out.println(bd2); + + long tt3 = 0; + BigDecimal bd3 = null; + for (int i = 0; i < LOOP; i++) + { + final long t3 = System.nanoTime(); + bd3 = extractDecimal(array3, 18, 6); + tt3 += System.nanoTime() - t3; + } + System.out.print(tt3 / LOOP); + System.out.print("ns >> "); + System.out.println(bd3); + + long tt4 = 0; + BigDecimal bd4 = null; + for (int i = 0; i < LOOP; i++) + { + final long t4 = System.nanoTime(); + bd4 = extractDecimal(array4, 18, 6); + tt4 += System.nanoTime() - t4; + } + System.out.print(tt4 / LOOP); + System.out.print("ns >> "); + System.out.println(bd4); + + long tt5 = 0; + BigDecimal bd5 = null; + for (int i = 0; i < LOOP; i++) + { + final long t5 = System.nanoTime(); + bd5 = extractDecimal(array5, 18, 6); + tt5 += System.nanoTime() - t5; + } + System.out.print(tt5 / LOOP); + System.out.print("ns >> "); + System.out.println(bd5); + } + while (false); + + do + { + System.out.println("new extract decimal: "); + + LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length); + LogBuffer buffer2 = new LogBuffer(array1, 0, array1.length); + LogBuffer buffer3 = new LogBuffer(array3, 0, array3.length); + LogBuffer buffer4 = new LogBuffer(array4, 0, array4.length); + LogBuffer buffer5 = new LogBuffer(array5, 0, array5.length); + + long tt1 = 0; + BigDecimal bd1 = null; + for (int i = 0; i < LOOP; i++) + { + final long t1 = System.nanoTime(); + bd1 = buffer1.getDecimal(0, 19, 10); + tt1 += System.nanoTime() - t1; + } + System.out.print(tt1 / LOOP); + System.out.print("ns >> "); + System.out.println(bd1); + + long tt2 = 0; + BigDecimal bd2 = null; + for (int i = 0; i < LOOP; i++) + { + final long t2 = System.nanoTime(); + bd2 = buffer2.getDecimal(0, 19, 10); + tt2 += System.nanoTime() - t2; + } + System.out.print(tt2 / LOOP); + System.out.print("ns >> "); + System.out.println(bd2); + + long tt3 = 0; + BigDecimal bd3 = null; + for (int i = 0; i < LOOP; i++) + { + final long t3 = System.nanoTime(); + bd3 = buffer3.getDecimal(0, 18, 6); + tt3 += System.nanoTime() - t3; + } + System.out.print(tt3 / LOOP); + System.out.print("ns >> "); + System.out.println(bd3); + + long tt4 = 0; + BigDecimal bd4 = null; + for (int i = 0; i < LOOP; i++) + { + final long t4 = System.nanoTime(); + bd4 = buffer4.getDecimal(0, 18, 6); + tt4 += System.nanoTime() - t4; + } + System.out.print(tt4 / LOOP); + System.out.print("ns >> "); + System.out.println(bd4); + + long tt5 = 0; + BigDecimal bd5 = null; + for (int i = 0; i < LOOP; i++) + { + final long t5 = System.nanoTime(); + bd5 = buffer5.getDecimal(0, 18, 6); + tt5 += System.nanoTime() - t5; + } + System.out.print(tt5 / LOOP); + System.out.print("ns >> "); + System.out.println(bd5); + } + while (false); + } +} diff --git a/dbsync/src/test/resources/binlog/mysql-bin.000001 b/dbsync/src/test/resources/binlog/mysql-bin.000001 new file mode 100644 index 0000000000000000000000000000000000000000..943ae0a060330ca87ced4fc9c9dcf7138c4ff251 GIT binary patch literal 22920 zcmc(n3v?Xib;m!;l28rCvYcYavGCZbEI(ySmI-#S(MnpGwb!z;S{W&^kc7381$t$* z_A$?bP96|yC<40%M=2pu`Z%YBgg!Vx>50KWfRkbs5=aqAU6dvV4!DF8mr&IEzw_tg+hSbQ+vGZb6bJ=nXhqw`=#|Grq~y550q&Q1P;%z{hjssdG{a_Of?sYaE%xPamX z1qC@%$6u+>!BK)V9^#@6cksVdo>GhHzp9i%oPY9X4<$Nt3rMU$#o_})9jZFGJyZ{o z1xVUR<1}X){9CC!Gp!&I8%osm^u|=q?0Gpy6(4sYN-n3djQ*ognI*F*_7MV0YPam| zqKbHIXrT9CEWS4>QFhH>H;!0{tk2N64F92S_$7WTm48w@!k?GtK>6jv9LerznOY`} zY9hfvG-yNv)#0FFv@?HuS$p#6ZM{AH2M)CxE9X$Kr?cJY=})XIFJEQU?~WSvP2uob z1OOh<;O)l2j`+U)9r2Z$HmovggLQ$X@N{ZJB(y6KX)$&NTa1+uTs3Ev5v-@K2DgU# z`v+>PrKIMHK%_Ak-FhI=b?JAWwai;Lxm4;MFD`lBAj0#unaUqvk!5D5lJj<<-F9;J zP{^}aN$D?hkIh1vz4_}CeS_aaSV*e@KWqk|>1I$PW}e`K)xOO|>eZJmW@hE$#p^wr zfryyd!Xh&>gTh6#=$K&|U!y{+nW>L5yGeh!0y73JG5KwV_-}18dj3z%)5csn*5A@3 z6{^$s!7ABvtwr4tb=o%H`wogFW!@EMv(0PhWsn_D=*F9 z@oiLIo)3|A_(ebZ99oFIXzh-gJ$v4~+@tEK%11S?VN?%Lb)l}FrDji^#dK6d{9;>b#U$Lj50IP{KZZx9i0bI*gW2XClm zZ=0yDnyPKy_(YPvK~W@@TKSN@+S1d7dhKGm9^}x9`@uBBwae255wY|Oua;2FyR@DT zT1M~1R!dW7vv|tzI`31ciPsR`+0~~$#&4zS_Pjd%_I@XaD!*rjX0%V)D@4TWi!9<{ z`cN%i7yp5~SN4>_E5D0CO}uU*HKii_XQeK<(&F`c4qfKiD@4TWqb%aVE2_n7{v3C& z>?wm+?g7-q>oQWKYeDVxbc@An7l&FrdxeO2{Vt1m@QP~j`rZt@I?gN9#OvJi;q|!o z`sNocULzdp^z0QP;`L8i#DiBxwT~yl%SyZRqyw6(ZvG zMi%kl71iSPd8&5v>Nu}Z6R(rZa#DN!xz+3Hx$qkI>=h#7bw7)E@QP~j`VUm?=GAe( zLQTA$VwN-7>x+9W^Lo*R@OqPHuMiQhQ5Nyw71iSPL8^B1>Nu}Z6R-coEQ{vjIzy>< zTPvxKzQ!ZwT7pK<8VJbQ(RcwIE#qgPam*Dk7d^XfRS zP!q4WGE1BGdfe*u84itl_6ia4iZ5O~%`2+KYlN!ZygJS+)WmBavwT^5J!SR!0}egl z*(*fE>lv?JQ7vA#EpYehIImC>uPvme)Q#HfZ>(Ow#i0j1dxeO2eVs)->{nEa*VUhL z_v$#WP!q2~Qln4(+Uvr0%YOX^hsHd6g@|~4jzv6pMYVXn=nvh!I?gN9#A`XJDRsB@ zdV|&LK@L6S*(*fE>))}62d}7RuU8RQ(M73V-9Ce#wI$x2k)q6?|X^sf`v}r)2(H9&Z`01;=Gi!mHM%Ee$wi^mqU+ub`BA7 z{?{zx!8xkgc|CERO>>_9N#1ccLruIcSOl-HX|H#8Sax$Who1246(ZvGD2sUTifZxt zCRMw=BRTG7sEOApW_d?@ebDN)_S0y?lb*doM7$2OhzGBz7OyW+wVPMRd4-yIeT!Mn zYOim1S>|;EZn&s3p1nduy!Nw*2d}6WuTN05n^(tqg_?N%DYMiR=}!h$uS;<2M!oOZ z>$_I3Ut|#vUQsPx|B|ZRygJS+)Wqvk%rdOK&hNI&YrztD{lK$Vh{(Lwv4{t+s1~pH zQni~`$9aXCczuXj{zQ9SYW4a)hkonXD@4R=MKK!Y!7Hl8>n&96=GAdtp(b9BGs_X} z^`c(Oy#9hiYyJYTUcNnshF@TPW=o1Jh>~%~H8jXE2Wiz1QqVU3uQVS8m-wpW5=d zJ>{sBHF2COm1_7ebK=)pZT*TvBff1xOl%b|!*Segp-609NkukWxZ+CMs;IQtsz9aK ziu2)qsBN7*tY-^1h5nI4U$|lxfM#|>lW&wjMC`oHBJOrj#Jjnfj#W$R!fI#gjFTUG z@R;2!bj55N=_)nv1JjYs>~}eIj%HtD5szleKbvY6c*)62U6W^UmpvKWn6hAPX2YaQ z%Gzx79?Q($&7s>o&n!e_W}jgZce5yxnZ5XrQq4N~aWY#8U76XrdA6OVqTjc!*K$;f zt)KAWey43UT5WxHC2T$G+ZM#cRx^va+d`4p8lfVaE!(pFlFe2HD#g~{@ZoZPYrdQO zYtmMb486ypQ`5Ha0kA5|Y*{XFAR=~3S5eUnt^`mdC)huxBCDO0E1Z;RvY%j6g3uMS z_cP2T+UzSyv$(4JPY#VwoAtOfAtt7mte%-^6p85s71>SO-c?e3*-fwefomau&$&aR*}3u1);W*!|>&B5BvNROHYuyRQm+yPALG z&@Ovr%C(EVvJynnuG4JeIo+<8{@l{8^<`KP3w*Z=V$!Zo7ISYGilkj1P?1Bs?7mXk z_3#T0?Xp*ZRqF^u^ON>+!q+F`3~$7IQa^BH0L2RODbf#h3lO5qr^LBiJib zHiEsf5=62#7FVDrd1uXYX^XX8-5grtyIl~Ic72n@+}njBZr8npwPB-UyX?MF*2wpD zyUx>JLfN;Ay)>m?sik0&hV3LKx&x~lcH!WMyZHf^PI~bn#{J46vzPQj<}-)+fQ#B5 z3Dif8Ksbyq2ls@+!R^6DV|Tr=x@@%(-EEY{2L=+QRi!1RMag6AB*_mw9iyxFz|ek+ zerr(^K ziKMFNYs;A%>88KcpD;RldkuQN=^#BxL(h;H1O43h#DJymMo%Yw58Tt$6YDgtzRnD= zkFO^W4j*@TI(Vv&rnp`|jU@F3NfpS1(7E4f{lsI5UZuYRuMN=iM1jVjffq%xog}wi z;-xiOuBkB?VX4%qgnX&!A0z#rc+xkQd}A;itck8QBEg1ms3s5%(V}k*2lfQ*FV%!Y zw2o{$0*A+raON5&uIA5q@onb{)suhXYigjr5>zem;@`pMU`-P}yG6L|=tO&Rqb9MW zGlN9%oDYJg#=v&gmvcZ;+THPvOaDI|C&=-WpEyCr$(Js-(lI}?+3I>GTc$R$yFovG z7pkK<4K+p^d9kbU9z=v5Z<-B)_enUEdgu1c;YT0?}|=byHnk zFako`eX>yPDuW(ZoWb*o($~jn+75sI-1~~P;M`Q|OMQJXoVl&32I(2homuP&`c;Jf zmS^;g%Nb2AYp!hyg!d%hU*0G6pI>%PjXBhLpre;>u+l`OK9_2wE;_?#HgNZp9t-Om zl*J*pP-~hZ5n7yWvOIA6GjFUWNZ1foaN!wvq{fk^1Nm%{ZjvYoq0`0Cy30~ z{N^=T^1A|TE%Vu9Xz?{HK3+Go7VeSaCzrGGxGVJJ>Us1Rv?zHY4ur3pZ>HozWhTpW ztH{8RtI7LVa@l5l_ED;TNM8+reDd7Lp(W%++aRBv9LapujSKW@O16@l4@?7_pPWW6 zhInXY@fF&`hZN!~;wwmMj6*Nz?$ZmIyKlcSZFU|y!awf9k7VJ`p@+WH`GL&*$i)!; zVmu_S_G$i)6Bhm$hkopbA3DN6!aCXek7VILk&z$B%#U0Q;h$jfxaOatko3Q#3f6q_ zLr3@{tdpG|$-+NO$?5$EGV>!BL-KKRQ1p|XAIZXBP08u} zKxTgAVhDdLi$Au-+{3ddB>f-bkS~7d2>-YXKazz%r#cfqkeMI37{Xsn`SeWj=S}_` z3JHIVL%#T-Bm5(*lfC~)7XA~IoZf#RGd}~;AB2B`#g}f)fekgALc(8CgO_~qLr3@{ ztdpG|$-+NO$?5#SW`5*i2>&RHH*Pceb15YJlm7XkBm70R=tFjXBny8vC8zTPnfZ~6 zA^fc@e*5Jne;$Q|e~d%E`VSr9A9vwLvhe57x47y2KxTgAVhDdRzV%g)Rhj&EQAqe> z96IIa{0|-BA7P#B{YSF!pUB7$WadXMhVW0Y_(;IyAEA)&m(;8viQB4 z{}_dYe~d%E`VSr9A9vwLvhe57l~+1HkeMI37{XtSD>ZdQ^B<>>@W(jhiyu0|Kf*fM z`;TPdKar6i$jpyi4B?+(@uQmm?kk6kgxtj zNBGBG_>nC9IrOz`IzNz^AGsL9UyQGD)pMHvNeT&nj6=Top(FewtdqU}NEZGR8To>v)g@nHZA0+*)f9ME*gmtp>BU$)|DLK9WKxTgAVhH~zi$AaVpQe!T zPx|MFj_?=Z_MgN0N3!r&Q*t^#keMI37{cGm;xB0a?O(O-YXKazz%haMJ7=La(LBNs#Xiz#2JjveOp!ww2b|6?5T#Sb0fA7P#B>j%lg ze zzvgeCknm6X=ZB8)7d4;{+4+$y{MD44&JSefM=pl&x3c&*HUBV$gnx`f2mSOPI>JBh z!jEL(&-p?oejqbHaxsLznDUkSwdVgSg@ixGAz%E^5&jX@$=-h?3;&6X{6J=Yk8#KsKXinD+=U;>!k-h( z#1CZVM=pl&7gN4cKhylnDJ1+c4*BATj_{ALPWJVKWZ^%Nksrv+k6aAlpJ4HS)BGhA z68@4VSo6gX9pR6#PIi7I3;!@Br>`F%Ge2@MgnyL9U)TIADJ1-p9P-5v9pNw9gFa;E zN3!r&Q*t^#keMI37{cGm;uD&G4TXe%j6=Top(Fg`F8oLq{v3LeAe|q`%#U0Q;V;IM z2kH&Y-$x1ko vGV>!BL-<=+yiM~r-AP}M;(C7#joUbsi-Xfo-*rP@?p;U!^X0M)_^J3mo1HN{ literal 0 HcmV?d00001 diff --git a/dbsync/src/test/resources/dummy.txt b/dbsync/src/test/resources/dummy.txt new file mode 100644 index 00000000..f77ca914 --- /dev/null +++ b/dbsync/src/test/resources/dummy.txt @@ -0,0 +1 @@ +本文件仅仅为定位绝对路径使用 \ No newline at end of file diff --git a/deployer/pom.xml b/deployer/pom.xml new file mode 100644 index 00000000..09c434d9 --- /dev/null +++ b/deployer/pom.xml @@ -0,0 +1,118 @@ + + 4.0.0 + + com.alibaba.otter + canal + 1.0.19-SNAPSHOT + ../pom.xml + + com.alibaba.otter + canal.deployer + jar + canal deployer module for otter ${project.version} + + + com.alibaba.otter + canal.server + ${project.version} + + + + + + + + maven-jar-plugin + + + true + + + **/logback.xml + **/canal.properties + **/spring/** + **/example/** + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + 2.2.1 + + + assemble + + single + + package + + + + false + false + + + + + + + + dev + + true + + env + !release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/dev.xml + + canal + ${project.build.directory} + + + + + + + + + release + + + env + release + + + + + + + maven-assembly-plugin + + + + ${basedir}/src/main/assembly/release.xml + + + ${project.artifactId}-${project.version} + + ${project.parent.build.directory} + + + + + + + diff --git a/deployer/src/main/assembly/dev.xml b/deployer/src/main/assembly/dev.xml new file mode 100644 index 00000000..8f1096c5 --- /dev/null +++ b/deployer/src/main/assembly/dev.xml @@ -0,0 +1,54 @@ + + dist + + dir + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/conf + /conf + + **/* + + + + ./src/main/resources + /conf + + **/* + + + + target + logs + + **/* + + + + + + lib + + junit:junit + + + + diff --git a/deployer/src/main/assembly/release.xml b/deployer/src/main/assembly/release.xml new file mode 100644 index 00000000..aada5f5d --- /dev/null +++ b/deployer/src/main/assembly/release.xml @@ -0,0 +1,54 @@ + + dist + + tar.gz + + false + + + . + / + + README* + + + + ./src/main/bin + bin + + **/* + + 0755 + + + ./src/main/conf + /conf + + **/* + + + + ./src/main/resources + /conf + + **/* + + + + target + logs + + **/* + + + + + + lib + + junit:junit + + + + diff --git a/deployer/src/main/bin/startup.bat b/deployer/src/main/bin/startup.bat new file mode 100755 index 00000000..0974ea41 --- /dev/null +++ b/deployer/src/main/bin/startup.bat @@ -0,0 +1,25 @@ +@echo off +@if not "%ECHO%" == "" echo %ECHO% +@if "%OS%" == "Windows_NT" setlocal + +set ENV_PATH=.\ +if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0% + +set conf_dir=%ENV_PATH%\..\conf +set canal_conf=%conf_dir%\canal.properties +set logback_configurationFile=%conf_dir%\logback.xml + +set CLASSPATH=%conf_dir% +set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH% + +set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m +set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 +set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n +set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%" + +set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS% + +set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher +echo start cmd : %CMD_STR% + +java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher \ No newline at end of file diff --git a/deployer/src/main/bin/startup.sh b/deployer/src/main/bin/startup.sh new file mode 100644 index 00000000..5655c713 --- /dev/null +++ b/deployer/src/main/bin/startup.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +current_path=`pwd` +case "`uname`" in + Linux) + bin_abs_path=$(readlink -f $(dirname $0)) + ;; + *) + bin_abs_path=`cd $(dirname $0); pwd` + ;; +esac +base=${bin_abs_path}/.. +canal_conf=$base/conf/canal.properties +logback_configurationFile=$base/conf/logback.xml +export LANG=en_US.UTF-8 +export BASE=$base + +if [ -f $base/bin/canal.pid ] ; then + echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2 + exit 1 +fi + +if [ ! -d $base/logs/canal ] ; then + mkdir -p $base/logs/canal +fi + +## set java path +if [ -z "$JAVA" ] ; then + JAVA=$(which java) +fi + +ALIBABA_JAVA="/usr/alibaba/java/bin/java" +TAOBAO_JAVA="/opt/taobao/java/bin/java" +if [ -z "$JAVA" ]; then + if [ -f $ALIBABA_JAVA ] ; then + JAVA=$ALIBABA_JAVA + elif [ -f $TAOBAO_JAVA ] ; then + JAVA=$TAOBAO_JAVA + else + echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2 + exit 1 + fi +fi + +case "$#" +in +0 ) + ;; +1 ) + var=$* + if [ -f $var ] ; then + canal_conf=$var + else + echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN." + exit + fi;; +2 ) + var=$1 + if [ -f $var ] ; then + canal_conf=$var + else + if [ "$1" = "debug" ]; then + DEBUG_PORT=$2 + DEBUG_SUSPEND="n" + JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND" + fi + fi;; +* ) + echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN." + exit;; +esac + +str=`file -L $JAVA | grep 64-bit` +if [ -n "$str" ]; then + JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError" +else + JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m " +fi + +JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8" +CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf" + +if [ -e $canal_conf -a -e $logback_configurationFile ] +then + + for i in $base/lib/*; + do CLASSPATH=$i:"$CLASSPATH"; + done + CLASSPATH="$base/conf:$CLASSPATH"; + + echo "cd to $bin_abs_path for workaround relative path" + cd $bin_abs_path + + echo LOG CONFIGURATION : $logback_configurationFile + echo canal conf : $canal_conf + echo CLASSPATH :$CLASSPATH + $JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 & + echo $! > $base/bin/canal.pid + + echo "cd to $current_path for continue" + cd $current_path +else + echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!" +fi diff --git a/deployer/src/main/bin/stop.sh b/deployer/src/main/bin/stop.sh new file mode 100644 index 00000000..73eeb9ca --- /dev/null +++ b/deployer/src/main/bin/stop.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +cygwin=false; +case "`uname`" in + CYGWIN*) + cygwin=true + ;; +esac + +get_pid() { + STR=$1 + PID=$2 + if $cygwin; then + JAVA_CMD="$JAVA_HOME\bin\java" + JAVA_CMD=`cygpath --path --unix $JAVA_CMD` + JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` + else + if [ ! -z "$PID" ]; then + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` + fi + fi + echo $JAVA_PID; +} + +base=`dirname $0`/.. +pidfile=$base/bin/canal.pid +if [ ! -f "$pidfile" ];then + echo "canal is not running. exists" + exit +fi + +pid=`cat $pidfile` +if [ "$pid" == "" ] ; then + pid=`get_pid "appName=otter-canal"` +fi + +echo -e "`hostname`: stopping canal $pid ... " +kill $pid + +LOOPS=0 +while (true); +do + gpid=`get_pid "appName=otter-canal" "$pid"` + if [ "$gpid" == "" ] ; then + echo "Oook! cost:$LOOPS" + `rm $pidfile` + break; + fi + let LOOPS=LOOPS+1 + sleep 1 +done \ No newline at end of file diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java new file mode 100644 index 00000000..f02942ae --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java @@ -0,0 +1,50 @@ +package com.alibaba.otter.canal.deployer; + +import java.text.MessageFormat; + +/** + * 启动常用变量 + * + * @author jianghang 2012-11-8 下午03:15:55 + * @version 1.0.0 + */ +public class CanalConstants { + + public static final String MDC_DESTINATION = "destination"; + public static final String ROOT = "canal"; + public static final String CANAL_ID = ROOT + "." + "id"; + public static final String CANAL_IP = ROOT + "." + "ip"; + public static final String CANAL_PORT = ROOT + "." + "port"; + public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers"; + + public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations"; + public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan"; + public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval"; + public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir"; + + public static final String CANAL_DESTINATION_SPLIT = ","; + public static final String GLOBAL_NAME = "global"; + + public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode"; + public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy"; + public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address"; + public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml"; + + public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination"; + + public static String getInstanceModeKey(String destination) { + return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination); + } + + public static String getInstanceManagerAddressKey(String destination) { + return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination); + } + + public static String getInstancSpringXmlKey(String destination) { + return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination); + } + + public static String getInstancLazyKey(String destination) { + return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination); + } +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java new file mode 100644 index 00000000..39c40cd5 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java @@ -0,0 +1,452 @@ +package com.alibaba.otter.canal.deployer; + +import java.util.Map; +import java.util.Properties; + +import org.I0Itec.zkclient.IZkStateListener; +import org.I0Itec.zkclient.exception.ZkNoNodeException; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import com.alibaba.otter.canal.common.utils.AddressUtils; +import com.alibaba.otter.canal.common.zookeeper.ZkClientx; +import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor; +import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors; +import com.alibaba.otter.canal.deployer.InstanceConfig.InstanceMode; +import com.alibaba.otter.canal.deployer.monitor.InstanceAction; +import com.alibaba.otter.canal.deployer.monitor.InstanceConfigMonitor; +import com.alibaba.otter.canal.deployer.monitor.ManagerInstanceConfigMonitor; +import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor; +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; +import com.alibaba.otter.canal.instance.manager.CanalConfigClient; +import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator; +import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator; +import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded; +import com.alibaba.otter.canal.server.exception.CanalServerException; +import com.alibaba.otter.canal.server.netty.CanalServerWithNetty; +import com.google.common.base.Function; +import com.google.common.collect.MapMaker; + +/** + * canal调度控制器 + * + * @author jianghang 2012-11-8 下午12:03:11 + * @version 1.0.0 + */ +public class CanalController { + + private static final Logger logger = LoggerFactory.getLogger(CanalController.class); + private Long cid; + private String ip; + private int port; + // 默认使用spring的方式载入 + private Map instanceConfigs; + private InstanceConfig globalInstanceConfig; + private Map managerClients; + // 监听instance config的变化 + private boolean autoScan = true; + private InstanceAction defaultAction; + private Map instanceConfigMonitors; + private CanalServerWithEmbeded embededCanalServer; + private CanalServerWithNetty canalServer; + + private CanalInstanceGenerator instanceGenerator; + private ZkClientx zkclientx; + + public CanalController(){ + this(System.getProperties()); + } + + public CanalController(final Properties properties){ + managerClients = new MapMaker().makeComputingMap(new Function() { + + public CanalConfigClient apply(String managerAddress) { + return getManagerClient(managerAddress); + } + }); + + // 初始化全局参数设置 + globalInstanceConfig = initGlobalConfig(properties); + instanceConfigs = new MapMaker().makeMap(); + // 初始化instance config + initInstanceConfig(properties); + + // 准备canal server + cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID)); + ip = getProperty(properties, CanalConstants.CANAL_IP); + port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT)); + embededCanalServer = new CanalServerWithEmbeded(); + embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator + canalServer = new CanalServerWithNetty(embededCanalServer); + canalServer.setIp(ip); + canalServer.setPort(port); + + // 处理下ip为空，默认使用hostIp暴露到zk中 + if (StringUtils.isEmpty(ip)) { + ip = AddressUtils.getHostIp(); + } + final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS); + if (StringUtils.isNotEmpty(zkServers)) { + zkclientx = ZkClientx.getZkClient(zkServers); + // 初始化系统目录 + zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true); + zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true); + } + + final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port); + ServerRunningMonitors.setServerData(serverData); + ServerRunningMonitors.setRunningMonitors(new MapMaker().makeComputingMap(new Function() { + + public ServerRunningMonitor apply(final String destination) { + ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData); + runningMonitor.setDestination(destination); + runningMonitor.setListener(new ServerRunningListener() { + + public void processActiveEnter() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + embededCanalServer.start(destination); + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processActiveExit() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + embededCanalServer.stop(destination); + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processStart() { + try { + if (zkclientx != null) { + final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":" + + port); + initCid(path); + zkclientx.subscribeStateChanges(new IZkStateListener() { + + public void handleStateChanged(KeeperState state) throws Exception { + + } + + public void handleNewSession() throws Exception { + initCid(path); + } + }); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + public void processStop() { + try { + MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination)); + if (zkclientx != null) { + final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":" + + port); + releaseCid(path); + } + } finally { + MDC.remove(CanalConstants.MDC_DESTINATION); + } + } + + }); + if (zkclientx != null) { + runningMonitor.setZkClient(zkclientx); + } + return runningMonitor; + } + })); + + // 初始化monitor机制 + autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN)); + if (autoScan) { + defaultAction = new InstanceAction() { + + public void start(String destination) { + InstanceConfig config = instanceConfigs.get(destination); + if (config == null) { + config = new InstanceConfig(globalInstanceConfig); + instanceConfigs.put(destination, config); + } + + if (!config.getLazy() && !embededCanalServer.isStart(destination)) { + // HA机制启动 + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (!runningMonitor.isStart()) { + runningMonitor.start(); + } + } + } + + public void stop(String destination) { + // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息 + InstanceConfig config = instanceConfigs.remove(destination); + if (config != null) { + embededCanalServer.stop(destination); + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } + } + + public void reload(String destination) { + // 目前任何配置变化，直接重启，简单处理 + stop(destination); + start(destination); + } + }; + + instanceConfigMonitors = new MapMaker().makeComputingMap(new Function() { + + public InstanceConfigMonitor apply(InstanceMode mode) { + int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL)); + + if (mode.isSpring()) { + SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor(); + monitor.setScanIntervalInSecond(scanInterval); + monitor.setDefaultAction(defaultAction); + // 设置conf目录，默认是user.dir + conf目录组成 + String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR); + if (StringUtils.isEmpty(rootDir)) { + rootDir = "../conf"; + } + monitor.setRootConf(rootDir); + return monitor; + } else if (mode.isManager()) { + return new ManagerInstanceConfigMonitor(); + } else { + throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor"); + } + } + }); + } + } + + private InstanceConfig initGlobalConfig(Properties properties) { + InstanceConfig globalConfig = new InstanceConfig(); + String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(modeStr)) { + globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); + } + + String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(lazyStr)) { + globalConfig.setLazy(Boolean.valueOf(lazyStr)); + } + + String managerAddress = getProperty(properties, + CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(managerAddress)) { + globalConfig.setManagerAddress(managerAddress); + } + + String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME)); + if (StringUtils.isNotEmpty(springXml)) { + globalConfig.setSpringXml(springXml); + } + + instanceGenerator = new CanalInstanceGenerator() { + + public CanalInstance generate(String destination) { + InstanceConfig config = instanceConfigs.get(destination); + if (config == null) { + throw new CanalServerException("can't find destination:{}"); + } + + if (config.getMode().isManager()) { + ManagerCanalInstanceGenerator instanceGenerator = new ManagerCanalInstanceGenerator(); + instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress())); + return instanceGenerator.generate(destination); + } else if (config.getMode().isSpring()) { + SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator(); + synchronized (this) { + try { + // 设置当前正在加载的通道，加载spring查找文件时会用到该变量 + System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination); + instanceGenerator.setBeanFactory(getBeanFactory(config.getSpringXml())); + return instanceGenerator.generate(destination); + } finally { + System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, ""); + } + } + } else { + throw new UnsupportedOperationException("unknow mode :" + config.getMode()); + } + + } + + }; + + return globalConfig; + } + + private CanalConfigClient getManagerClient(String managerAddress) { + return new CanalConfigClient(); + } + + private BeanFactory getBeanFactory(String springXml) { + ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml); + return applicationContext; + } + + private void initInstanceConfig(Properties properties) { + String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS); + String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT); + + for (String destination : destinations) { + InstanceConfig config = parseInstanceConfig(properties, destination); + InstanceConfig oldConfig = instanceConfigs.put(destination, config); + + if (oldConfig != null) { + logger.warn("destination:{} old config:{} has replace by new config:{}", new Object[] { destination, + oldConfig, config }); + } + } + } + + private InstanceConfig parseInstanceConfig(Properties properties, String destination) { + InstanceConfig config = new InstanceConfig(globalInstanceConfig); + String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination)); + if (!StringUtils.isEmpty(modeStr)) { + config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr))); + } + + String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination)); + if (!StringUtils.isEmpty(lazyStr)) { + config.setLazy(Boolean.valueOf(lazyStr)); + } + + if (config.getMode().isManager()) { + String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination)); + if (StringUtils.isNotEmpty(managerAddress)) { + config.setManagerAddress(managerAddress); + } + } else if (config.getMode().isSpring()) { + String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination)); + if (StringUtils.isNotEmpty(springXml)) { + config.setSpringXml(springXml); + } + } + + return config; + } + + private String getProperty(Properties properties, String key) { + return StringUtils.trim(properties.getProperty(StringUtils.trim(key))); + } + + public void start() throws Throwable { + logger.info("## start the canal server[{}:{}]", ip, port); + // 创建整个canal的工作节点 + final String path = ZookeeperPathUtils.getCanalClusterNode(ip + ":" + port); + initCid(path); + if (zkclientx != null) { + this.zkclientx.subscribeStateChanges(new IZkStateListener() { + + public void handleStateChanged(KeeperState state) throws Exception { + + } + + public void handleNewSession() throws Exception { + initCid(path); + } + }); + } + // 优先启动embeded服务 + embededCanalServer.start(); + // 尝试启动一下非lazy状态的通道 + for (Map.Entry entry : instanceConfigs.entrySet()) { + final String destination = entry.getKey(); + InstanceConfig config = entry.getValue(); + // 创建destination的工作节点 + if (!config.getLazy() && !embededCanalServer.isStart(destination)) { + // HA机制启动 + ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination); + if (!runningMonitor.isStart()) { + runningMonitor.start(); + } + } + + if (autoScan) { + instanceConfigMonitors.get(config.getMode()).regeister(destination, defaultAction); + } + } + + if (autoScan) { + instanceConfigMonitors.get(globalInstanceConfig.getMode()).start(); + for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { + if (!monitor.isStart()) { + monitor.start(); + } + } + } + + // 启动网络接口 + canalServer.start(); + } + + public void stop() throws Throwable { + canalServer.stop(); + + if (autoScan) { + for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) { + if (monitor.isStart()) { + monitor.stop(); + } + } + } + + for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) { + if (runningMonitor.isStart()) { + runningMonitor.stop(); + } + } + + // 释放canal的工作节点 + releaseCid(ZookeeperPathUtils.getCanalClusterNode(ip + ":" + port)); + logger.info("## stop the canal server[{}:{}]", ip, port); + } + + private void initCid(String path) { + // logger.info("## init the canalId = {}", cid); + // 初始化系统目录 + if (zkclientx != null) { + try { + zkclientx.createEphemeral(path); + } catch (ZkNoNodeException e) { + // 如果父目录不存在，则创建 + String parentDir = path.substring(0, path.lastIndexOf('/')); + zkclientx.createPersistent(parentDir, true); + zkclientx.createEphemeral(path); + } + + } + } + + private void releaseCid(String path) { + // logger.info("## release the canalId = {}", cid); + // 初始化系统目录 + if (zkclientx != null) { + zkclientx.delete(path); + } + } + +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java new file mode 100644 index 00000000..8f9e4371 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java @@ -0,0 +1,58 @@ +package com.alibaba.otter.canal.deployer; + +import java.io.FileInputStream; +import java.util.Properties; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * canal独立版本启动的入口类 + * + * @author jianghang 2012-11-6 下午05:20:49 + * @version 1.0.0 + */ +public class CanalLauncher { + + private static final String CLASSPATH_URL_PREFIX = "classpath:"; + private static final Logger logger = LoggerFactory.getLogger(CanalLauncher.class); + + public static void main(String[] args) throws Throwable { + try { + String conf = System.getProperty("canal.conf", "classpath:canal.properties"); + Properties properties = new Properties(); + if (conf.startsWith(CLASSPATH_URL_PREFIX)) { + conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX); + properties.load(CanalLauncher.class.getClassLoader().getResourceAsStream(conf)); + } else { + properties.load(new FileInputStream(conf)); + } + + logger.info("## start the canal server."); + final CanalController controller = new CanalController(properties); + controller.start(); + logger.info("## the canal server is running now ......"); + Runtime.getRuntime().addShutdownHook(new Thread() { + + public void run() { + try { + logger.info("## stop the canal server"); + controller.stop(); + } catch (Throwable e) { + logger.warn("##something goes wrong when stopping canal Server:\n{}", + ExceptionUtils.getFullStackTrace(e)); + } finally { + logger.info("## canal server is down."); + } + } + + }); + } catch (Throwable e) { + logger.error("## Something goes wrong when starting up the canal Server:\n{}", + ExceptionUtils.getFullStackTrace(e)); + System.exit(0); + } + } +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/InstanceConfig.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/InstanceConfig.java new file mode 100644 index 00000000..7ea2f882 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/InstanceConfig.java @@ -0,0 +1,93 @@ +package com.alibaba.otter.canal.deployer; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; + +/** + * 启动的相关配置 + * + * @author jianghang 2012-11-8 下午02:50:54 + * @version 1.0.0 + */ +public class InstanceConfig { + + private InstanceConfig globalConfig; + private InstanceMode mode; + private Boolean lazy; + private String managerAddress; + private String springXml; + + public InstanceConfig(){ + + } + + public InstanceConfig(InstanceConfig globalConfig){ + this.globalConfig = globalConfig; + } + + public static enum InstanceMode { + SPRING, MANAGER; + + public boolean isSpring() { + return this == InstanceMode.SPRING; + } + + public boolean isManager() { + return this == InstanceMode.MANAGER; + } + } + + public Boolean getLazy() { + if (lazy == null && globalConfig != null) { + return globalConfig.getLazy(); + } else { + return lazy; + } + } + + public void setLazy(Boolean lazy) { + this.lazy = lazy; + } + + public InstanceMode getMode() { + if (mode == null && globalConfig != null) { + return globalConfig.getMode(); + } else { + return mode; + } + } + + public void setMode(InstanceMode mode) { + this.mode = mode; + } + + public String getManagerAddress() { + if (managerAddress == null && globalConfig != null) { + return globalConfig.getManagerAddress(); + } else { + return managerAddress; + } + } + + public void setManagerAddress(String managerAddress) { + this.managerAddress = managerAddress; + } + + public String getSpringXml() { + if (springXml == null && globalConfig != null) { + return globalConfig.getSpringXml(); + } else { + return springXml; + } + } + + public void setSpringXml(String springXml) { + this.springXml = springXml; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java new file mode 100644 index 00000000..14551ea7 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceAction.java @@ -0,0 +1,25 @@ +package com.alibaba.otter.canal.deployer.monitor; + +/** + * config配置变化 + * + * @author jianghang 2013-2-18 下午01:19:29 + * @version 1.0.1 + */ +public interface InstanceAction { + + /** + * 启动destination + */ + public void start(String destination); + + /** + * 停止destination + */ + public void stop(String destination); + + /** + * 重载destination，可能需要stop,start操作，或者只是更新下内存配置 + */ + public void reload(String destination); +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java new file mode 100644 index 00000000..c1ec67d1 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/InstanceConfigMonitor.java @@ -0,0 +1,16 @@ +package com.alibaba.otter.canal.deployer.monitor; + +import com.alibaba.otter.canal.common.CanalLifeCycle; + +/** + * 监听instance file的文件变化，触发instance start/stop等操作 + * + * @author jianghang 2013-2-6 下午06:19:56 + * @version 1.0.1 + */ +public interface InstanceConfigMonitor extends CanalLifeCycle { + + public void regeister(String destination, InstanceAction action); + + public void unRegeister(String destination); +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java new file mode 100644 index 00000000..ee508b9f --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/ManagerInstanceConfigMonitor.java @@ -0,0 +1,20 @@ +package com.alibaba.otter.canal.deployer.monitor; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.CanalLifeCycle; + +/** + * @author jianghang 2013-2-18 下午03:19:06 + * @version 1.0.1 + */ +public class ManagerInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { + + public void regeister(String destination, InstanceAction action) { + + } + + public void unRegeister(String destination) { + + } + +} diff --git a/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java new file mode 100644 index 00000000..e19323a2 --- /dev/null +++ b/deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java @@ -0,0 +1,297 @@ +package com.alibaba.otter.canal.deployer.monitor; + +import java.io.File; +import java.io.FileFilter; +import java.io.FilenameFilter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.common.AbstractCanalLifeCycle; +import com.alibaba.otter.canal.common.CanalLifeCycle; +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.google.common.base.Function; +import com.google.common.collect.MapMaker; + +/** + * 监听基于spring配置的instance变化 + * + * @author jianghang 2013-2-6 下午06:23:55 + * @version 1.0.1 + */ +public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implements InstanceConfigMonitor, CanalLifeCycle { + + private static final Logger logger = LoggerFactory.getLogger(SpringInstanceConfigMonitor.class); + private String rootConf; + // 扫描周期，单位秒 + private long scanIntervalInSecond = 5; + private InstanceAction defaultAction = null; + private Map actions = new MapMaker().makeMap(); + private Map lastFiles = new MapMaker().makeComputingMap(new Function() { + + public InstanceConfigFiles apply(String destination) { + return new InstanceConfigFiles(destination); + } + }); + private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("canal-instance-scan")); + + public void start() { + super.start(); + Assert.notNull(rootConf, "root conf dir is null!"); + + executor.scheduleWithFixedDelay(new Runnable() { + + public void run() { + try { + scan(); + } catch (Throwable e) { + logger.error("scan failed", e); + } + } + + }, 0, scanIntervalInSecond, TimeUnit.SECONDS); + } + + public void stop() { + super.stop(); + executor.shutdownNow(); + actions.clear(); + lastFiles.clear(); + } + + public void regeister(String destination, InstanceAction action) { + if (action != null) { + actions.put(destination, action); + } else { + actions.put(destination, defaultAction); + } + } + + public void unRegeister(String destination) { + actions.remove(destination); + } + + public void setRootConf(String rootConf) { + this.rootConf = rootConf; + } + + private void scan() { + File rootdir = new File(rootConf); + if (!rootdir.exists()) { + return; + } + + File[] instanceDirs = rootdir.listFiles(new FileFilter() { + + public boolean accept(File pathname) { + String filename = pathname.getName(); + return pathname.isDirectory() && !"spring".equalsIgnoreCase(filename); + } + }); + + // 扫描目录的新增 + Set currentInstanceNames = new HashSet(); + + // 判断目录内文件的变化 + for (File instanceDir : instanceDirs) { + String destination = instanceDir.getName(); + currentInstanceNames.add(destination); + File[] instanceConfigs = instanceDir.listFiles(new FilenameFilter() { + + public boolean accept(File dir, String name) { + // return !StringUtils.endsWithIgnoreCase(name, ".dat"); + // 限制一下，只针对instance.properties文件,避免因为.svn或者其他生成的临时文件导致出现reload + return StringUtils.equalsIgnoreCase(name, "instance.properties"); + } + + }); + + if (!actions.containsKey(destination) && instanceConfigs.length > 0) { + // 存在合法的instance.properties，并且第一次添加时，进行启动操作 + notifyStart(instanceDir, destination); + } else if (actions.containsKey(destination)) { + // 历史已经启动过 + if (instanceConfigs.length == 0) { // 如果不存在合法的instance.properties + notifyStop(destination); + } else { + InstanceConfigFiles lastFile = lastFiles.get(destination); + boolean hasChanged = judgeFileChanged(instanceConfigs, lastFile.getInstanceFiles()); + // 通知变化 + if (hasChanged) { + notifyReload(destination); + } + + if (hasChanged || CollectionUtils.isEmpty(lastFile.getInstanceFiles())) { + // 更新内容 + List newFileInfo = new ArrayList(); + for (File instanceConfig : instanceConfigs) { + newFileInfo.add(new FileInfo(instanceConfig.getName(), instanceConfig.lastModified())); + } + + lastFile.setInstanceFiles(newFileInfo); + } + } + } + + } + + // 判断目录是否删除 + Set deleteInstanceNames = new HashSet(); + for (String destination : actions.keySet()) { + if (!currentInstanceNames.contains(destination)) { + deleteInstanceNames.add(destination); + } + } + for (String deleteInstanceName : deleteInstanceNames) { + notifyStop(deleteInstanceName); + } + } + + private void notifyStart(File instanceDir, String destination) { + try { + defaultAction.start(destination); + actions.put(destination, defaultAction); + logger.info("auto notify start {} successful.", destination); + } catch (Throwable e) { + logger.error("scan add found[{}] but start failed", destination, ExceptionUtils.getFullStackTrace(e)); + } + } + + private void notifyStop(String destination) { + InstanceAction action = actions.remove(destination); + try { + action.stop(destination); + logger.info("auto notify stop {} successful.", destination); + } catch (Throwable e) { + logger.error("scan delete found[{}] but stop failed", destination, ExceptionUtils.getFullStackTrace(e)); + actions.put(destination, action);// 再重新加回去，下一次scan时再执行删除 + } + } + + private void notifyReload(String destination) { + InstanceAction action = actions.get(destination); + if (action != null) { + try { + action.reload(destination); + logger.info("auto notify reload {} successful.", destination); + } catch (Throwable e) { + logger.error("scan reload found[{}] but reload failed", + destination, + ExceptionUtils.getFullStackTrace(e)); + } + } + } + + private boolean judgeFileChanged(File[] instanceConfigs, List fileInfos) { + boolean hasChanged = false; + for (File instanceConfig : instanceConfigs) { + for (FileInfo fileInfo : fileInfos) { + if (instanceConfig.getName().equals(fileInfo.getName())) { + hasChanged |= (instanceConfig.lastModified() != fileInfo.getLastModified()); + if (hasChanged) { + return hasChanged; + } + } + } + } + + return hasChanged; + } + + public void setDefaultAction(InstanceAction defaultAction) { + this.defaultAction = defaultAction; + } + + public void setScanIntervalInSecond(long scanIntervalInSecond) { + this.scanIntervalInSecond = scanIntervalInSecond; + } + + public static class InstanceConfigFiles { + + private String destination; // instance + // name + private List springFile = new ArrayList(); // spring的instance + // xml + private FileInfo rootFile; // canal.properties + private List instanceFiles = new ArrayList(); // instance对应的配置 + + public InstanceConfigFiles(String destination){ + this.destination = destination; + } + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public List getSpringFile() { + return springFile; + } + + public void setSpringFile(List springFile) { + this.springFile = springFile; + } + + public FileInfo getRootFile() { + return rootFile; + } + + public void setRootFile(FileInfo rootFile) { + this.rootFile = rootFile; + } + + public List getInstanceFiles() { + return instanceFiles; + } + + public void setInstanceFiles(List instanceFiles) { + this.instanceFiles = instanceFiles; + } + + } + + public static class FileInfo { + + private String name; + private long lastModified = 0; + + public FileInfo(String name, long lastModified){ + this.name = name; + this.lastModified = lastModified; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public long getLastModified() { + return lastModified; + } + + public void setLastModified(long lastModified) { + this.lastModified = lastModified; + } + + } + +} diff --git a/deployer/src/main/resources/canal.properties b/deployer/src/main/resources/canal.properties new file mode 100644 index 00000000..2831c4e1 --- /dev/null +++ b/deployer/src/main/resources/canal.properties @@ -0,0 +1,62 @@ +################################################# +######### common argument ############# +################################################# +canal.id= 1 +canal.ip= +canal.port= 11111 +canal.zkServers= +# flush data to zk +canal.zookeeper.flush.period = 1000 +# flush meta cursor/parse position to file +canal.file.data.dir = ${canal.conf.dir} +canal.file.flush.period = 1000 +## memory store RingBuffer size, should be Math.pow(2,n) +canal.instance.memory.buffer.size = 16384 +## memory store RingBuffer used memory unit size , default 1kb +canal.instance.memory.buffer.memunit = 1024 +## meory store gets mode used MEMSIZE or ITEMSIZE +canal.instance.memory.batch.mode = MEMSIZE + +## detecing config +canal.instance.detecting.enable = false +#canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now() +canal.instance.detecting.sql = select 1 +canal.instance.detecting.interval.time = 3 +canal.instance.detecting.retry.threshold = 3 +canal.instance.detecting.heartbeatHaEnable = false + +# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery +canal.instance.transaction.size = 1024 +# mysql fallback connected to new master should fallback times +canal.instance.fallbackIntervalInSeconds = 60 + +# network config +canal.instance.network.receiveBufferSize = 16384 +canal.instance.network.sendBufferSize = 16384 +canal.instance.network.soTimeout = 30 + +# binlog filter config +canal.instance.filter.query.dcl = false +canal.instance.filter.query.dml = false +canal.instance.filter.query.ddl = false +canal.instance.filter.table.error = false + +# binlog ddl isolation +canal.instance.get.ddl.isolation = false + +################################################# +######### destinations ############# +################################################# +canal.destinations= example +# conf root dir +canal.conf.dir = ../conf +# auto scan instance dir add/remove and start/stop instance +canal.auto.scan = true +canal.auto.scan.interval = 5 + +canal.instance.global.mode = spring +canal.instance.global.lazy = false +#canal.instance.global.manager.address = 127.0.0.1:1099 +#canal.instance.global.spring.xml = classpath:spring/memory-instance.xml +canal.instance.global.spring.xml = classpath:spring/file-instance.xml +#canal.instance.global.spring.xml = classpath:spring/default-instance.xml \ No newline at end of file diff --git a/deployer/src/main/resources/example/instance.properties b/deployer/src/main/resources/example/instance.properties new file mode 100644 index 00000000..2bdfffb2 --- /dev/null +++ b/deployer/src/main/resources/example/instance.properties @@ -0,0 +1,27 @@ +################################################# +## mysql serverId +canal.instance.mysql.slaveId = 1234 + +# position info +canal.instance.master.address = 127.0.0.1:3306 +canal.instance.master.journal.name = +canal.instance.master.position = +canal.instance.master.timestamp = + +#canal.instance.standby.address = +#canal.instance.standby.journal.name = +#canal.instance.standby.position = +#canal.instance.standby.timestamp = + +# username/password +canal.instance.dbUsername = canal +canal.instance.dbPassword = canal +canal.instance.defaultDatabaseName = +canal.instance.connectionCharset = UTF-8 + +# table regex +canal.instance.filter.regex = .*\\..* +# table black regex +canal.instance.filter.black.regex = + +################################################# \ No newline at end of file diff --git a/deployer/src/main/resources/logback.xml b/deployer/src/main/resources/logback.xml new file mode 100644 index 00000000..0d34e3f5 --- /dev/null +++ b/deployer/src/main/resources/logback.xml @@ -0,0 +1,83 @@ + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + destination + canal + + + + ../logs/${destination}/${destination}.log + + + ../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz + + + 512MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n + + + + + + + + + destination + canal + + + + ../logs/${destination}/meta.log + + + ../logs/${destination}/%d{yyyy-MM-dd}/meta-%d{yyyy-MM-dd}-%i.log.gz + + + 32MB + + 60 + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/default-instance.xml b/deployer/src/main/resources/spring/default-instance.xml new file mode 100644 index 00000000..1a1c3291 --- /dev/null +++ b/deployer/src/main/resources/spring/default-instance.xml @@ -0,0 +1,183 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${canal.zkServers:127.0.0.1:2181} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/file-instance.xml b/deployer/src/main/resources/spring/file-instance.xml new file mode 100644 index 00000000..74b48f98 --- /dev/null +++ b/deployer/src/main/resources/spring/file-instance.xml @@ -0,0 +1,168 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/group-instance.xml b/deployer/src/main/resources/spring/group-instance.xml new file mode 100644 index 00000000..40c7787b --- /dev/null +++ b/deployer/src/main/resources/spring/group-instance.xml @@ -0,0 +1,253 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deployer/src/main/resources/spring/memory-instance.xml b/deployer/src/main/resources/spring/memory-instance.xml new file mode 100644 index 00000000..524ecb3c --- /dev/null +++ b/deployer/src/main/resources/spring/memory-instance.xml @@ -0,0 +1,156 @@ + + + + + + + + + + classpath:canal.properties + classpath:${canal.instance.destination:}/instance.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/driver/pom.xml b/driver/pom.xml new file mode 100644 index 00000000..aa53b3cf --- /dev/null +++ b/driver/pom.xml @@ -0,0 +1,47 @@ + + 4.0.0 + + com.alibaba.otter + canal + 1.0.19-SNAPSHOT + ../pom.xml + + com.alibaba.otter + canal.parse.driver + jar + canal driver module for otter ${project.version} + + + com.alibaba.otter + canal.common + ${project.version} + + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + slf4j-api + + + org.jboss.netty + netty + + + + junit + junit + test + + + diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java new file mode 100644 index 00000000..9121048a --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnector.java @@ -0,0 +1,294 @@ +package com.alibaba.otter.canal.parse.driver.mysql; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.ClientAuthenticationPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.HandshakeInitializationPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.Reply323Packet; +import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter; +import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; + +/** + * 基于mysql socket协议的链接实现 + * + * @author jianghang 2013-2-18 下午09:22:30 + * @version 1.0.1 + */ +public class MysqlConnector { + + private static final Logger logger = LoggerFactory.getLogger(MysqlConnector.class); + private InetSocketAddress address; + private String username; + private String password; + + private byte charsetNumber = 33; + private String defaultSchema = "retl"; + private int soTimeout = 30 * 1000; + private int receiveBufferSize = 16 * 1024; + private int sendBufferSize = 16 * 1024; + + private SocketChannel channel; + private AtomicBoolean connected = new AtomicBoolean(false); + + public MysqlConnector(){ + } + + public MysqlConnector(InetSocketAddress address, String username, String password){ + + this.address = address; + this.username = username; + this.password = password; + } + + public MysqlConnector(InetSocketAddress address, String username, String password, byte charsetNumber, + String defaultSchema){ + this(address, username, password); + + this.charsetNumber = charsetNumber; + this.defaultSchema = defaultSchema; + } + + public void connect() throws IOException { + if (connected.compareAndSet(false, true)) { + try { + channel = SocketChannel.open(); + configChannel(channel); + logger.info("connect MysqlConnection to {}...", address); + channel.connect(address); + negotiate(channel); + } catch (Exception e) { + disconnect(); + throw new IOException("connect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e)); + } + } else { + logger.error("the channel can't be connected twice."); + } + } + + public void reconnect() throws IOException { + disconnect(); + connect(); + } + + public void disconnect() throws IOException { + if (connected.compareAndSet(true, false)) { + try { + if (channel != null) { + channel.close(); + } + + logger.info("disConnect MysqlConnection to {}...", address); + } catch (Exception e) { + throw new IOException("disconnect " + this.address + " failure:" + ExceptionUtils.getStackTrace(e)); + } + } else { + logger.info("the channel {} is not connected", this.address); + } + } + + public boolean isConnected() { + return this.channel != null && this.channel.isConnected(); + } + + public MysqlConnector fork() { + MysqlConnector connector = new MysqlConnector(); + connector.setCharsetNumber(getCharsetNumber()); + connector.setDefaultSchema(getDefaultSchema()); + connector.setAddress(getAddress()); + connector.setPassword(password); + connector.setUsername(getUsername()); + connector.setReceiveBufferSize(getReceiveBufferSize()); + connector.setSendBufferSize(getSendBufferSize()); + connector.setSoTimeout(getSoTimeout()); + return connector; + } + + // ====================== help method ==================== + + private void configChannel(SocketChannel channel) throws IOException { + channel.socket().setKeepAlive(true); + channel.socket().setReuseAddress(true); + channel.socket().setSoTimeout(soTimeout); + channel.socket().setTcpNoDelay(true); + channel.socket().setReceiveBufferSize(receiveBufferSize); + channel.socket().setSendBufferSize(sendBufferSize); + } + + private void negotiate(SocketChannel channel) throws IOException { + HeaderPacket header = PacketManager.readHeader(channel, 4); + byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength()); + if (body[0] < 0) {// check field_count + if (body[0] == -1) { + ErrorPacket error = new ErrorPacket(); + error.fromBytes(body); + throw new IOException("handshake exception:\n" + error.toString()); + } else if (body[0] == -2) { + throw new IOException("Unexpected EOF packet at handshake phase."); + } else { + throw new IOException("unpexpected packet with field_count=" + body[0]); + } + } + HandshakeInitializationPacket handshakePacket = new HandshakeInitializationPacket(); + handshakePacket.fromBytes(body); + + logger.info("handshake initialization packet received, prepare the client authentication packet to send"); + + ClientAuthenticationPacket clientAuth = new ClientAuthenticationPacket(); + clientAuth.setCharsetNumber(charsetNumber); + + clientAuth.setUsername(username); + clientAuth.setPassword(password); + clientAuth.setServerCapabilities(handshakePacket.serverCapabilities); + clientAuth.setDatabaseName(defaultSchema); + clientAuth.setScrumbleBuff(joinAndCreateScrumbleBuff(handshakePacket)); + + byte[] clientAuthPkgBody = clientAuth.toBytes(); + HeaderPacket h = new HeaderPacket(); + h.setPacketBodyLength(clientAuthPkgBody.length); + h.setPacketSequenceNumber((byte) (header.getPacketSequenceNumber() + 1)); + + PacketManager.write(channel, + new ByteBuffer[] { ByteBuffer.wrap(h.toBytes()), ByteBuffer.wrap(clientAuthPkgBody) }); + logger.info("client authentication packet is sent out."); + + // check auth result + header = null; + header = PacketManager.readHeader(channel, 4); + body = null; + body = PacketManager.readBytes(channel, header.getPacketBodyLength()); + assert body != null; + if (body[0] < 0) { + if (body[0] == -1) { + ErrorPacket err = new ErrorPacket(); + err.fromBytes(body); + throw new IOException("Error When doing Client Authentication:" + err.toString()); + } else if (body[0] == -2) { + auth323(channel, header.getPacketSequenceNumber(), handshakePacket.seed); + // throw new + // IOException("Unexpected EOF packet at Client Authentication."); + } else { + throw new IOException("unpexpected packet with field_count=" + body[0]); + } + } + } + + private void auth323(SocketChannel channel, byte packetSequenceNumber, byte[] seed) throws IOException { + // auth 323 + Reply323Packet r323 = new Reply323Packet(); + if (password != null && password.length() > 0) { + r323.seed = MySQLPasswordEncrypter.scramble323(password, new String(seed)).getBytes(); + } + byte[] b323Body = r323.toBytes(); + + HeaderPacket h323 = new HeaderPacket(); + h323.setPacketBodyLength(b323Body.length); + h323.setPacketSequenceNumber((byte) (packetSequenceNumber + 1)); + + PacketManager.write(channel, new ByteBuffer[] { ByteBuffer.wrap(h323.toBytes()), ByteBuffer.wrap(b323Body) }); + logger.info("client 323 authentication packet is sent out."); + // check auth result + HeaderPacket header = PacketManager.readHeader(channel, 4); + byte[] body = PacketManager.readBytes(channel, header.getPacketBodyLength()); + assert body != null; + switch (body[0]) { + case 0: + break; + case -1: + ErrorPacket err = new ErrorPacket(); + err.fromBytes(body); + throw new IOException("Error When doing Client Authentication:" + err.toString()); + default: + throw new IOException("unpexpected packet with field_count=" + body[0]); + } + } + + private byte[] joinAndCreateScrumbleBuff(HandshakeInitializationPacket handshakePacket) throws IOException { + byte[] dest = new byte[handshakePacket.seed.length + handshakePacket.restOfScrambleBuff.length]; + System.arraycopy(handshakePacket.seed, 0, dest, 0, handshakePacket.seed.length); + System.arraycopy(handshakePacket.restOfScrambleBuff, + 0, + dest, + handshakePacket.seed.length, + handshakePacket.restOfScrambleBuff.length); + return dest; + } + + public InetSocketAddress getAddress() { + return address; + } + + public void setAddress(InetSocketAddress address) { + this.address = address; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public byte getCharsetNumber() { + return charsetNumber; + } + + public void setCharsetNumber(byte charsetNumber) { + this.charsetNumber = charsetNumber; + } + + public String getDefaultSchema() { + return defaultSchema; + } + + public void setDefaultSchema(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + + public int getSoTimeout() { + return soTimeout; + } + + public void setSoTimeout(int soTimeout) { + this.soTimeout = soTimeout; + } + + public int getReceiveBufferSize() { + return receiveBufferSize; + } + + public void setReceiveBufferSize(int receiveBufferSize) { + this.receiveBufferSize = receiveBufferSize; + } + + public int getSendBufferSize() { + return sendBufferSize; + } + + public void setSendBufferSize(int sendBufferSize) { + this.sendBufferSize = sendBufferSize; + } + + public SocketChannel getChannel() { + return channel; + } + + public void setChannel(SocketChannel channel) { + this.channel = channel; + } + + public void setPassword(String password) { + this.password = password; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java new file mode 100644 index 00000000..07ea21a2 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlQueryExecutor.java @@ -0,0 +1,107 @@ +package com.alibaba.otter.canal.parse.driver.mysql; + +import java.io.IOException; +import java.nio.channels.SocketChannel; +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.RowDataPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; + +/** + * 默认输出的数据编码为UTF-8，如有需要请正确转码 + * + * @author jianghang 2013-9-4 上午11:50:26 + * @since 1.0.0 + */ +public class MysqlQueryExecutor { + + private SocketChannel channel; + + public MysqlQueryExecutor(MysqlConnector connector){ + if (!connector.isConnected()) { + throw new RuntimeException("should execute connector.connect() first"); + } + + this.channel = connector.getChannel(); + } + + public MysqlQueryExecutor(SocketChannel ch){ + this.channel = ch; + } + + /** + * (Result Set Header Packet) the number of columns
+ * (Field Packets) column descriptors
+ * (EOF Packet) marker: end of Field Packets
+ * (Row Data Packets) row contents
+ * (EOF Packet) marker: end of Data Packets + * + * @param queryString + * @return + * @throws IOException + */ + public ResultSetPacket query(String queryString) throws IOException { + QueryCommandPacket cmd = new QueryCommandPacket(); + cmd.setQueryString(queryString); + byte[] bodyBytes = cmd.toBytes(); + PacketManager.write(channel, bodyBytes); + byte[] body = readNextPacket(); + + if (body[0] < 0) { + ErrorPacket packet = new ErrorPacket(); + packet.fromBytes(body); + throw new IOException(packet + "\n with command: " + queryString); + } + + ResultSetHeaderPacket rsHeader = new ResultSetHeaderPacket(); + rsHeader.fromBytes(body); + + List fields = new ArrayList(); + for (int i = 0; i < rsHeader.getColumnCount(); i++) { + FieldPacket fp = new FieldPacket(); + fp.fromBytes(readNextPacket()); + fields.add(fp); + } + + readEofPacket(); + + List rowData = new ArrayList(); + while (true) { + body = readNextPacket(); + if (body[0] == -2) { + break; + } + RowDataPacket rowDataPacket = new RowDataPacket(); + rowDataPacket.fromBytes(body); + rowData.add(rowDataPacket); + } + + ResultSetPacket resultSet = new ResultSetPacket(); + resultSet.getFieldDescriptors().addAll(fields); + for (RowDataPacket r : rowData) { + resultSet.getFieldValues().addAll(r.getColumns()); + } + resultSet.setSourceAddress(channel.socket().getRemoteSocketAddress()); + + return resultSet; + } + + private void readEofPacket() throws IOException { + byte[] eofBody = readNextPacket(); + if (eofBody[0] != -2) { + throw new IOException("EOF Packet is expected, but packet with field_count=" + eofBody[0] + " is found."); + } + } + + protected byte[] readNextPacket() throws IOException { + HeaderPacket h = PacketManager.readHeader(channel, 4); + return PacketManager.readBytes(channel, h.getPacketBodyLength()); + } +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java new file mode 100644 index 00000000..521b07a0 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlUpdateExecutor.java @@ -0,0 +1,56 @@ +package com.alibaba.otter.canal.parse.driver.mysql; + +import java.io.IOException; +import java.nio.channels.SocketChannel; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.client.QueryCommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ErrorPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.server.OKPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager; + +/** + * 默认输出的数据编码为UTF-8，如有需要请正确转码 + * + * @author jianghang 2013-9-4 上午11:51:11 + * @since 1.0.0 + */ +public class MysqlUpdateExecutor { + + private static final Logger logger = LoggerFactory.getLogger(MysqlUpdateExecutor.class); + + private SocketChannel channel; + + public MysqlUpdateExecutor(MysqlConnector connector){ + if (!connector.isConnected()) { + throw new RuntimeException("should execute connector.connect() first"); + } + + this.channel = connector.getChannel(); + } + + public MysqlUpdateExecutor(SocketChannel ch){ + this.channel = ch; + } + + public OKPacket update(String updateString) throws IOException { + QueryCommandPacket cmd = new QueryCommandPacket(); + cmd.setQueryString(updateString); + byte[] bodyBytes = cmd.toBytes(); + PacketManager.write(channel, bodyBytes); + + logger.debug("read update result..."); + byte[] body = PacketManager.readBytes(channel, PacketManager.readHeader(channel, 4).getPacketBodyLength()); + if (body[0] < 0) { + ErrorPacket packet = new ErrorPacket(); + packet.fromBytes(body); + throw new IOException(packet + "\n with command: " + updateString); + } + + OKPacket packet = new OKPacket(); + packet.fromBytes(body); + return packet; + } +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/CommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/CommandPacket.java new file mode 100644 index 00000000..c264d89a --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/CommandPacket.java @@ -0,0 +1,24 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; + +public abstract class CommandPacket implements IPacket { + + private byte command; + + // arg + + public void setCommand(byte command) { + this.command = command; + } + + public byte getCommand() { + return command; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java new file mode 100644 index 00000000..c4dc7bd1 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/HeaderPacket.java @@ -0,0 +1,73 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; + +/** + * 
+ * Offset  Length     Description
+ *   0       3        Packet body length stored with the low byte first.
+ *   3       1        Packet sequence number. The sequence numbers are reset with each new command. 
+ *                      While the correct packet sequencing is ensured by the underlying transmission protocol,
+ *                      this field is used for the sanity checks of the application logic.
+ *
+ * + *
+ * The Packet Header will not be shown in the descriptions of packets that follow this section. Think of it as always + * there. But logically, it "precedes the packet" rather than "is included in the packet".
+ * + * @author fujohnwang + */ +public class HeaderPacket implements IPacket { + + /** + * this field indicates the packet length that follows the header, with header packet's 4 bytes excluded. + */ + private int packetBodyLength; + private byte packetSequenceNumber; + + /** + * little-endian byte order + */ + public byte[] toBytes() { + byte[] data = new byte[4]; + data[0] = (byte) (packetBodyLength & 0xFF); + data[1] = (byte) (packetBodyLength >>> 8); + data[2] = (byte) (packetBodyLength >>> 16); + data[3] = getPacketSequenceNumber(); + return data; + } + + /** + * little-endian byte order + */ + public void fromBytes(byte[] data) { + if (data == null || data.length != 4) { + throw new IllegalArgumentException("invalid header data. It can't be null and the length must be 4 byte."); + } + this.packetBodyLength = (data[0] & 0xFF) | ((data[1] & 0xFF) << 8) | ((data[2] & 0xFF) << 16); + this.setPacketSequenceNumber(data[3]); + } + + public int getPacketBodyLength() { + return packetBodyLength; + } + + public void setPacketBodyLength(int packetBodyLength) { + this.packetBodyLength = packetBodyLength; + } + + public void setPacketSequenceNumber(byte packetSequenceNumber) { + this.packetSequenceNumber = packetSequenceNumber; + } + + public byte getPacketSequenceNumber() { + return packetSequenceNumber; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java new file mode 100644 index 00000000..41ce2c96 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/IPacket.java @@ -0,0 +1,29 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets; + +import java.io.IOException; + +/** + * Top Abstraction for network packet.
+ * it exposes 2 behaviors for sub-class implementation which will be used to + * marshal data into bytes before sending and to un-marshal data from data after + * receiving.
+ * + * @author fujohnwang + * @see 1.0 + */ +public interface IPacket { + /** + * un-marshal raw bytes into {@link IPacket} state for application usage.
+ * + * @param data, the raw byte data received from networking + */ + void fromBytes(byte[] data) throws IOException; + + /** + * marshal the {@link IPacket} state into raw bytes for sending out to + * network.
+ * + * @return the bytes that's collected from {@link IPacket} state + */ + byte[] toBytes() throws IOException; +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/PacketWithHeaderPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/PacketWithHeaderPacket.java new file mode 100644 index 00000000..959cfa5f --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/PacketWithHeaderPacket.java @@ -0,0 +1,32 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets; + +import org.apache.commons.lang.builder.ToStringBuilder; + +import com.alibaba.otter.canal.common.utils.CanalToStringStyle; +import com.google.common.base.Preconditions; + +public abstract class PacketWithHeaderPacket implements IPacket { + + protected HeaderPacket header; + + protected PacketWithHeaderPacket(){ + } + + protected PacketWithHeaderPacket(HeaderPacket header){ + setHeader(header); + } + + public void setHeader(HeaderPacket header) { + Preconditions.checkNotNull(header); + this.header = header; + } + + public HeaderPacket getHeader() { + return header; + } + + public String toString() { + return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java new file mode 100644 index 00000000..f0fba11a --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/BinlogDumpCommandPacket.java @@ -0,0 +1,70 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.client; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.apache.commons.lang.StringUtils; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +/** + * COM_BINLOG_DUMP + * + * @author fujohnwang + * @since 1.0 + */ +public class BinlogDumpCommandPacket extends CommandPacket { + + /** BINLOG_DUMP options */ + public static final int BINLOG_DUMP_NON_BLOCK = 1; + public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2; + public long binlogPosition; + public long slaveServerId; + public String binlogFileName; + + public BinlogDumpCommandPacket(){ + setCommand((byte) 0x12); + } + + public void fromBytes(byte[] data) { + // bypass + } + + /** + * 
+     * Bytes                        Name
+     *  -----                        ----
+     *  1                            command
+     *  n                            arg
+     *  --------------------------------------------------------
+     *  Bytes                        Name
+     *  -----                        ----
+     *  4                            binlog position to start at (little endian)
+     *  2                            binlog flags (currently not used; always 0)
+     *  4                            server_id of the slave (little endian)
+     *  n                            binlog file name (optional)
+     * 
+     *
+ */ + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + // 0. write command number + out.write(getCommand()); + // 1. write 4 bytes bin-log position to start at + ByteHelper.writeUnsignedIntLittleEndian(binlogPosition, out); + // 2. write 2 bytes bin-log flags + int binlog_flags = 0; + binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT; + out.write(binlog_flags); + out.write(0x00); + // 3. write 4 bytes server id of the slave + ByteHelper.writeUnsignedIntLittleEndian(this.slaveServerId, out); + // 4. write bin-log file name if necessary + if (StringUtils.isNotEmpty(this.binlogFileName)) { + out.write(this.binlogFileName.getBytes()); + } + return out.toByteArray(); + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java new file mode 100644 index 00000000..04975a31 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/ClientAuthenticationPacket.java @@ -0,0 +1,130 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.client; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.lang.StringUtils; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; +import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC; +import com.alibaba.otter.canal.parse.driver.mysql.utils.MySQLPasswordEncrypter; + +public class ClientAuthenticationPacket extends PacketWithHeaderPacket { + + private String username; + private String password; + private byte charsetNumber; + private String databaseName; + private int serverCapabilities; + private byte[] scrumbleBuff; + + public void fromBytes(byte[] data) { + // bypass since nowhere to use. + } + + /** + * 
+     * VERSION 4.1
+     *  Bytes                        Name
+     *  -----                        ----
+     *  4                            client_flags
+     *  4                            max_packet_size
+     *  1                            charset_number
+     *  23                           (filler) always 0x00...
+     *  n (Null-Terminated String)   user
+     *  n (Length Coded Binary)      scramble_buff (1 + x bytes)
+     *  n (Null-Terminated String)   databasename (optional)
+     *
+ * + * @throws IOException + */ + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + // 1. write client_flags + // 1|4|512|1024|8192|32768 + /** + * CLIENT_LONG_PASSWORD CLIENT_LONG_FLAG CLIENT_PROTOCOL_41 + * CLIENT_INTERACTIVE CLIENT_TRANSACTIONS CLIENT_SECURE_CONNECTION + */ + ByteHelper.writeUnsignedIntLittleEndian(1 | 4 | 512 | 8192 | 32768, out); // remove + // client_interactive + // feature + + // 2. write max_packet_size + ByteHelper.writeUnsignedIntLittleEndian(MSC.MAX_PACKET_LENGTH, out); + // 3. write charset_number + out.write(this.charsetNumber); + // 4. write (filler) always 0x00... + out.write(new byte[23]); + // 5. write (Null-Terminated String) user + ByteHelper.writeNullTerminatedString(getUsername(), out); + // 6. write (Length Coded Binary) scramble_buff (1 + x bytes) + if (StringUtils.isEmpty(getPassword())) { + out.write(0x00); + } else { + try { + byte[] encryptedPassword = MySQLPasswordEncrypter.scramble411(getPassword().getBytes(), scrumbleBuff); + ByteHelper.writeBinaryCodedLengthBytes(encryptedPassword, out); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("can't encrypt password that will be sent to MySQL server.", e); + } + } + // 7 . (Null-Terminated String) databasename (optional) + if (getDatabaseName() != null) { + ByteHelper.writeNullTerminatedString(getDatabaseName(), out); + } + // end write + return out.toByteArray(); + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public void setCharsetNumber(byte charsetNumber) { + this.charsetNumber = charsetNumber; + } + + public byte getCharsetNumber() { + return charsetNumber; + } + + public void setDatabaseName(String databaseName) { + this.databaseName = databaseName; + } + + public String getDatabaseName() { + return databaseName; + } + + public void setServerCapabilities(int serverCapabilities) { + this.serverCapabilities = serverCapabilities; + } + + public int getServerCapabilities() { + return serverCapabilities; + } + + public void setScrumbleBuff(byte[] scrumbleBuff) { + this.scrumbleBuff = scrumbleBuff; + } + + public byte[] getScrumbleBuff() { + return scrumbleBuff; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QueryCommandPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QueryCommandPacket.java new file mode 100644 index 00000000..74ff3a63 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/client/QueryCommandPacket.java @@ -0,0 +1,34 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.client; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; + +public class QueryCommandPacket extends CommandPacket { + + private String queryString; + + public QueryCommandPacket(){ + setCommand((byte) 0x03); + } + + public void fromBytes(byte[] data) throws IOException { + } + + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(getCommand()); + out.write(getQueryString().getBytes("UTF-8"));// 链接建立时默认指定编码为UTF-8 + return out.toByteArray(); + } + + public void setQueryString(String queryString) { + this.queryString = queryString; + } + + public String getQueryString() { + return queryString; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/DataPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/DataPacket.java new file mode 100644 index 00000000..424295a3 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/DataPacket.java @@ -0,0 +1,17 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.CommandPacket; + +public class DataPacket extends CommandPacket { + + public void fromBytes(byte[] data) { + + } + + public byte[] toBytes() throws IOException { + return null; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/EOFPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/EOFPacket.java new file mode 100644 index 00000000..6b200c2e --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/EOFPacket.java @@ -0,0 +1,46 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +public class EOFPacket extends PacketWithHeaderPacket { + + public byte fieldCount; + public int warningCount; + public int statusFlag; + + /** + * 
+     *  VERSION 4.1
+     *  Bytes                 Name
+     *  -----                 ----
+     *  1                     field_count, always = 0xfe
+     *  2                     warning_count
+     *  2                     Status Flags
+     *
+ */ + public void fromBytes(byte[] data) { + int index = 0; + // 1. read field count + fieldCount = data[index]; + index++; + // 2. read warning count + this.warningCount = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // 3. read status flag + this.statusFlag = ByteHelper.readUnsignedShortLittleEndian(data, index); + // end read + } + + public byte[] toBytes() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(5); + out.write(this.fieldCount); + ByteHelper.writeUnsignedShortLittleEndian(this.warningCount, out); + ByteHelper.writeUnsignedShortLittleEndian(this.statusFlag, out); + return out.toByteArray(); + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ErrorPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ErrorPacket.java new file mode 100644 index 00000000..a9dfc778 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ErrorPacket.java @@ -0,0 +1,66 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +public class ErrorPacket extends PacketWithHeaderPacket { + + public byte fieldCount; + public int errorNumber; + public byte sqlStateMarker; + public byte[] sqlState; + public String message; + + /** + * 
+     * VERSION 4.1
+     *  Bytes                       Name
+     *  -----                       ----
+     *  1                           field_count, always = 0xff
+     *  2                           errno
+     *  1                           (sqlstate marker), always '#'
+     *  5                           sqlstate (5 characters)
+     *  n                           message
+     * 
+     *
+ */ + public void fromBytes(byte[] data) { + int index = 0; + // 1. read field count + this.fieldCount = data[0]; + index++; + // 2. read error no. + this.errorNumber = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // 3. read marker + this.sqlStateMarker = data[index]; + index++; + // 4. read sqlState + this.sqlState = ByteHelper.readFixedLengthBytes(data, index, 5); + index += 5; + // 5. read message + this.message = new String(ByteHelper.readFixedLengthBytes(data, index, data.length - index)); + // end read + } + + public byte[] toBytes() throws IOException { + return null; + } + + @Override + public String toString() { + return "ErrorPacket [errorNumber=" + errorNumber + ", fieldCount=" + fieldCount + ", message=" + message + + ", sqlState=" + sqlStateToString() + ", sqlStateMarker=" + (char) sqlStateMarker + "]"; + } + + private String sqlStateToString() { + StringBuilder builder = new StringBuilder(5); + for (byte b : this.sqlState) { + builder.append((char) b); + } + return builder.toString(); + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java new file mode 100644 index 00000000..984bc745 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/FieldPacket.java @@ -0,0 +1,190 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; +import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader; + +public class FieldPacket extends PacketWithHeaderPacket { + + private String catalog; + private String db; + private String table; + private String originalTable; + private String name; + private String originalName; + private int character; + private long length; + private byte type; + private int flags; + private byte decimals; + private String definition; + + /** + * 
+     *  VERSION 4.1
+     *  Bytes                      Name
+     *  -----                      ----
+     *  n (Length Coded String)    catalog
+     *  n (Length Coded String)    db
+     *  n (Length Coded String)    table
+     *  n (Length Coded String)    org_table
+     *  n (Length Coded String)    name
+     *  n (Length Coded String)    org_name
+     *  1                          (filler)
+     *  2                          charsetnr
+     *  4                          length
+     *  1                          type
+     *  2                          flags
+     *  1                          decimals
+     *  2                          (filler), always 0x00
+     *  n (Length Coded Binary)    default
+     * 
+     *
+ */ + public void fromBytes(byte[] data) throws IOException { + + int index = 0; + LengthCodedStringReader reader = new LengthCodedStringReader(null, index); + // 1. + catalog = reader.readLengthCodedString(data); + // 2. + db = reader.readLengthCodedString(data); + this.table = reader.readLengthCodedString(data); + this.originalTable = reader.readLengthCodedString(data); + this.name = reader.readLengthCodedString(data); + this.originalName = reader.readLengthCodedString(data); + index = reader.getIndex(); + // + index++; + // + this.character = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // + this.length = ByteHelper.readUnsignedIntLittleEndian(data, index); + index += 4; + // + this.type = data[index]; + index++; + // + this.flags = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // + this.decimals = data[index]; + index++; + // + if (index < data.length) { + reader.setIndex(index); + this.definition = reader.readLengthCodedString(data); + } + } + + public byte[] toBytes() throws IOException { + return null; + } + + public String getCatalog() { + return catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public String getDb() { + return db; + } + + public void setDb(String db) { + this.db = db; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getOriginalTable() { + return originalTable; + } + + public void setOriginalTable(String originalTable) { + this.originalTable = originalTable; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getOriginalName() { + return originalName; + } + + public void setOriginalName(String originalName) { + this.originalName = originalName; + } + + public int getCharacter() { + return character; + } + + public void setCharacter(int character) { + this.character = character; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + public byte getType() { + return type; + } + + public void setType(byte type) { + this.type = type; + } + + public int getFlags() { + return flags; + } + + public void setFlags(int flags) { + this.flags = flags; + } + + public byte getDecimals() { + return decimals; + } + + public void setDecimals(byte decimals) { + this.decimals = decimals; + } + + public String getDefinition() { + return definition; + } + + public void setDefinition(String definition) { + this.definition = definition; + } + + public String toString() { + return "FieldPacket [catalog=" + catalog + ", character=" + character + ", db=" + db + ", decimals=" + decimals + + ", definition=" + definition + ", flags=" + flags + ", length=" + length + ", name=" + name + + ", originalName=" + originalName + ", originalTable=" + originalTable + ", table=" + table + ", type=" + + type + "]"; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/HandshakeInitializationPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/HandshakeInitializationPacket.java new file mode 100644 index 00000000..d46b884b --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/HandshakeInitializationPacket.java @@ -0,0 +1,91 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; +import com.alibaba.otter.canal.parse.driver.mysql.utils.MSC; + +/** + * MySQL Handshake Initialization Packet.
+ * + * @author fujohnwang + * @since 1.0 + */ +public class HandshakeInitializationPacket extends PacketWithHeaderPacket { + + public byte protocolVersion = MSC.DEFAULT_PROTOCOL_VERSION; + public String serverVersion; + public long threadId; + public byte[] seed; + public int serverCapabilities; + public byte serverCharsetNumber; + public int serverStatus; + public byte[] restOfScrambleBuff; + + public HandshakeInitializationPacket(){ + } + + public HandshakeInitializationPacket(HeaderPacket header){ + super(header); + } + + /** + * 
+     * Bytes                        Name
+     *  -----                        ----
+     *  1                            protocol_version
+     *  n (Null-Terminated String)   server_version
+     *  4                            thread_id
+     *  8                            scramble_buff
+     *  1                            (filler) always 0x00
+     *  2                            server_capabilities
+     *  1                            server_language
+     *  2                            server_status
+     *  13                           (filler) always 0x00 ...
+     *  13                           rest of scramble_buff (4.1)
+     *
+ */ + public void fromBytes(byte[] data) { + int index = 0; + // 1. read protocol_version + protocolVersion = data[index]; + index++; + // 2. read server_version + byte[] serverVersionBytes = ByteHelper.readNullTerminatedBytes(data, index); + serverVersion = new String(serverVersionBytes); + index += (serverVersionBytes.length + 1); + // 3. read thread_id + threadId = ByteHelper.readUnsignedIntLittleEndian(data, index); + index += 4; + // 4. read scramble_buff + seed = ByteHelper.readFixedLengthBytes(data, index, 8); + index += 8; + index += 1; // 1 byte (filler) always 0x00 + // 5. read server_capabilities + this.serverCapabilities = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // 6. read server_language + this.serverCharsetNumber = data[index]; + index++; + // 7. read server_status + this.serverStatus = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // 8. bypass filtered bytes + index += 13; + // 9. read rest of scramble_buff + this.restOfScrambleBuff = ByteHelper.readFixedLengthBytes(data, index, 12); // 虽然Handshake Initialization + // Packet规定最后13个byte是剩下的scrumble, + // 但实际上最后一个字节是0, 不应该包含在scrumble中. + // end read + } + + /** + * Bypass implementing it, 'cause nowhere to use it. + */ + public byte[] toBytes() throws IOException { + return null; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/OKPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/OKPacket.java new file mode 100644 index 00000000..fcd3173d --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/OKPacket.java @@ -0,0 +1,118 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.IOException; +import java.util.Arrays; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +/** + * Aka. OK packet + * + * @author fujohnwang + */ +public class OKPacket extends PacketWithHeaderPacket { + + public byte fieldCount; + public byte[] affectedRows; + public byte[] insertId; + public int serverStatus; + public int warningCount; + public String message; + + /** + * 
+     *  VERSION 4.1
+     *  Bytes                       Name
+     *  -----                       ----
+     *  1   (Length Coded Binary)   field_count, always = 0
+     *  1-9 (Length Coded Binary)   affected_rows
+     *  1-9 (Length Coded Binary)   insert_id
+     *  2                           server_status
+     *  2                           warning_count
+     *  n   (until end of packet)   message
+     *
+ * + * @throws IOException + */ + public void fromBytes(byte[] data) throws IOException { + int index = 0; + // 1. read field count + this.fieldCount = data[0]; + index++; + // 2. read affected rows + this.affectedRows = ByteHelper.readBinaryCodedLengthBytes(data, index); + index += this.affectedRows.length; + // 3. read insert id + this.insertId = ByteHelper.readBinaryCodedLengthBytes(data, index); + index += this.insertId.length; + // 4. read server status + this.serverStatus = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // 5. read warning count + this.warningCount = ByteHelper.readUnsignedShortLittleEndian(data, index); + index += 2; + // 6. read message. + this.message = new String(ByteHelper.readFixedLengthBytes(data, index, data.length - index)); + // end read + } + + public byte[] toBytes() throws IOException { + return null; + } + + public byte getFieldCount() { + return fieldCount; + } + + public void setFieldCount(byte fieldCount) { + this.fieldCount = fieldCount; + } + + public byte[] getAffectedRows() { + return affectedRows; + } + + public void setAffectedRows(byte[] affectedRows) { + this.affectedRows = affectedRows; + } + + public byte[] getInsertId() { + return insertId; + } + + public void setInsertId(byte[] insertId) { + this.insertId = insertId; + } + + public int getServerStatus() { + return serverStatus; + } + + public void setServerStatus(int serverStatus) { + this.serverStatus = serverStatus; + } + + public int getWarningCount() { + return warningCount; + } + + public void setWarningCount(int warningCount) { + this.warningCount = warningCount; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String toString() { + return "OKPacket [affectedRows=" + Arrays.toString(affectedRows) + ", fieldCount=" + fieldCount + ", insertId=" + + Arrays.toString(insertId) + ", message=" + message + ", serverStatus=" + serverStatus + + ", warningCount=" + warningCount + "]"; + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/Reply323Packet.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/Reply323Packet.java new file mode 100644 index 00000000..2a1b5f7c --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/Reply323Packet.java @@ -0,0 +1,27 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +public class Reply323Packet extends PacketWithHeaderPacket { + + public byte[] seed; + + public void fromBytes(byte[] data) throws IOException { + + } + + public byte[] toBytes() throws IOException { + if (seed == null) { + return new byte[] { (byte) 0 }; + } else { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ByteHelper.writeNullTerminated(seed, out); + return out.toByteArray(); + } + } + +} diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetHeaderPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetHeaderPacket.java new file mode 100644 index 00000000..b6cdc9a7 --- /dev/null +++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetHeaderPacket.java @@ -0,0 +1,67 @@ +package com.alibaba.otter.canal.parse.driver.mysql.packets.server; + +import java.io.IOException; + +import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket; +import com.alibaba.otter.canal.parse.driver.mysql.utils.ByteHelper; + +/** + * 
+ * Type Of Result Packet       Hexadecimal Value Of First Byte (field_count)
+ * ---------------------------------------------------------------------------
+ * Result Set Packet           1-250 (first byte of Length-Coded Binary)
+ *
+ * + * The sequence of result set packet: + * + * 
+ *   (Result Set Header Packet)  the number of columns
+ *   (Field Packets)             column descriptors
+ *   (EOF Packet)                marker: end of Field Packets
+ *   (Row Data Packets)          row contents
+ * (EOF Packet)                marker: end of Data Packets
+ * 
+ * 
+ * 
+ * @author fujohnwang
+ */
+public class ResultSetHeaderPacket extends PacketWithHeaderPacket {
+
+    private long columnCount;
+    private long extra;
+
+    public void fromBytes(byte[] data) throws IOException {
+        int index = 0;
+        byte[] colCountBytes = ByteHelper.readBinaryCodedLengthBytes(data, index);
+        columnCount = ByteHelper.readLengthCodedBinary(colCountBytes, index);
+        index += colCountBytes.length;
+        if (index < data.length - 1) {
+            extra = ByteHelper.readLengthCodedBinary(data, index);
+        }
+    }
+
+    public byte[] toBytes() throws IOException {
+        return null;
+    }
+
+    public long getColumnCount() {
+        return columnCount;
+    }
+
+    public void setColumnCount(long columnCount) {
+        this.columnCount = columnCount;
+    }
+
+    public long getExtra() {
+        return extra;
+    }
+
+    public void setExtra(long extra) {
+        this.extra = extra;
+    }
+
+    public String toString() {
+        return "ResultSetHeaderPacket [columnCount=" + columnCount + ", extra=" + extra + "]";
+    }
+
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java
new file mode 100644
index 00000000..ed538dee
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/ResultSetPacket.java
@@ -0,0 +1,42 @@
+package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
+
+import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ResultSetPacket {
+
+    private SocketAddress     sourceAddress;
+    private List fieldDescriptors = new ArrayList();
+    private List      fieldValues      = new ArrayList();
+
+    public void setFieldDescriptors(List fieldDescriptors) {
+        this.fieldDescriptors = fieldDescriptors;
+    }
+
+    public List getFieldDescriptors() {
+        return fieldDescriptors;
+    }
+
+    public void setFieldValues(List fieldValues) {
+        this.fieldValues = fieldValues;
+    }
+
+    public List getFieldValues() {
+        return fieldValues;
+    }
+
+    public void setSourceAddress(SocketAddress sourceAddress) {
+        this.sourceAddress = sourceAddress;
+    }
+
+    public SocketAddress getSourceAddress() {
+        return sourceAddress;
+    }
+
+    public String toString() {
+        return "ResultSetPacket [fieldDescriptors=" + fieldDescriptors + ", fieldValues=" + fieldValues
+               + ", sourceAddress=" + sourceAddress + "]";
+    }
+
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java
new file mode 100644
index 00000000..1f6a45eb
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/packets/server/RowDataPacket.java
@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.parse.driver.mysql.packets.server;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.LengthCodedStringReader;
+
+public class RowDataPacket extends PacketWithHeaderPacket {
+
+    private List columns = new ArrayList();
+
+    public void fromBytes(byte[] data) throws IOException {
+        int index = 0;
+        LengthCodedStringReader reader = new LengthCodedStringReader(null, index);
+        do {
+            getColumns().add(reader.readLengthCodedString(data));
+        } while (reader.getIndex() < data.length);
+    }
+
+    public byte[] toBytes() throws IOException {
+        return null;
+    }
+
+    public void setColumns(List columns) {
+        this.columns = columns;
+    }
+
+    public List getColumns() {
+        return columns;
+    }
+
+    public String toString() {
+        return "RowDataPacket [columns=" + columns + "]";
+    }
+
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/BinlogDumpCommandBuilder.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/BinlogDumpCommandBuilder.java
new file mode 100644
index 00000000..d483eaa6
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/BinlogDumpCommandBuilder.java
@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.io.IOException;
+
+import org.apache.commons.lang.StringUtils;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+
+public class BinlogDumpCommandBuilder {
+
+    public BinlogDumpCommandPacket build(String binglogFile, long position, long slaveId) {
+        BinlogDumpCommandPacket command = new BinlogDumpCommandPacket();
+        command.binlogPosition = position;
+        if (!StringUtils.isEmpty(binglogFile)) {
+            command.binlogFileName = binglogFile;
+        }
+        command.slaveServerId = slaveId;
+        // end settings.
+        return command;
+    }
+
+    public ChannelBuffer toChannelBuffer(BinlogDumpCommandPacket command) throws IOException {
+        byte[] commandBytes = command.toBytes();
+        byte[] headerBytes = assembleHeaderBytes(commandBytes.length);
+        ChannelBuffer buffer = ChannelBuffers.wrappedBuffer(headerBytes, commandBytes);
+        return buffer;
+    }
+
+    private byte[] assembleHeaderBytes(int length) {
+        HeaderPacket header = new HeaderPacket();
+        header.setPacketBodyLength(length);
+        header.setPacketSequenceNumber((byte) 0x00);
+        return header.toBytes();
+    }
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java
new file mode 100644
index 00000000..a5e5a3ac
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ByteHelper.java
@@ -0,0 +1,158 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public abstract class ByteHelper {
+
+    public static final long NULL_LENGTH = -1;
+
+    public static byte[] readNullTerminatedBytes(byte[] data, int index) {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        for (int i = index; i < data.length; i++) {
+            byte item = data[i];
+            if (item == MSC.NULL_TERMINATED_STRING_DELIMITER) {
+                break;
+            }
+            out.write(item);
+        }
+        return out.toByteArray();
+    }
+
+    public static void writeNullTerminatedString(String str, ByteArrayOutputStream out) throws IOException {
+        out.write(str.getBytes());
+        out.write(MSC.NULL_TERMINATED_STRING_DELIMITER);
+    }
+    
+    public static void writeNullTerminated(byte[] data, ByteArrayOutputStream out) throws IOException {
+        out.write(data);
+        out.write(MSC.NULL_TERMINATED_STRING_DELIMITER);
+    }
+
+    public static byte[] readFixedLengthBytes(byte[] data, int index, int length) {
+        byte[] bytes = new byte[length];
+        System.arraycopy(data, index, bytes, 0, length);
+        return bytes;
+    }
+
+    /**
+     * Read 4 bytes in Little-endian byte order.
+     * 
+     * @param data, the original byte array
+     * @param index, start to read from.
+     * @return
+     */
+    public static long readUnsignedIntLittleEndian(byte[] data, int index) {
+        long result = (long) (data[index] & 0xFF) | (long) ((data[index + 1] & 0xFF) << 8)
+                      | (long) ((data[index + 2] & 0xFF) << 16) | (long) ((data[index + 3] & 0xFF) << 24);
+        return result;
+    }
+
+    public static long readUnsignedLongLittleEndian(byte[] data, int index) {
+        long accumulation = 0;
+        int position = index;
+        for (int shiftBy = 0; shiftBy < 64; shiftBy += 8) {
+            accumulation |= (long) ((data[position++] & 0xff) << shiftBy);
+        }
+        return accumulation;
+    }
+
+    public static int readUnsignedShortLittleEndian(byte[] data, int index) {
+        int result = (data[index] & 0xFF) | ((data[index + 1] & 0xFF) << 8);
+        return result;
+    }
+
+    public static int readUnsignedMediumLittleEndian(byte[] data, int index) {
+        int result = (data[index] & 0xFF) | ((data[index + 1] & 0xFF) << 8) | ((data[index + 2] & 0xFF) << 16);
+        return result;
+    }
+
+    public static long readLengthCodedBinary(byte[] data, int index) throws IOException {
+        int firstByte = data[index] & 0xFF;
+        switch (firstByte) {
+            case 251:
+                return NULL_LENGTH;
+            case 252:
+                return readUnsignedShortLittleEndian(data, index + 1);
+            case 253:
+                return readUnsignedMediumLittleEndian(data, index + 1);
+            case 254:
+                return readUnsignedLongLittleEndian(data, index + 1);
+            default:
+                return firstByte;
+        }
+    }
+
+    public static byte[] readBinaryCodedLengthBytes(byte[] data, int index) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        out.write(data[index]);
+
+        byte[] buffer = null;
+        int value = data[index] & 0xFF;
+        if (value == 251) {
+            buffer = new byte[0];
+        }
+        if (value == 252) {
+            buffer = new byte[2];
+        }
+        if (value == 253) {
+            buffer = new byte[3];
+        }
+        if (value == 254) {
+            buffer = new byte[8];
+        }
+        if (buffer != null) {
+            System.arraycopy(data, index + 1, buffer, 0, buffer.length);
+            out.write(buffer);
+        }
+
+        return out.toByteArray();
+    }
+
+    public static void writeUnsignedIntLittleEndian(long data, ByteArrayOutputStream out) {
+        out.write((byte) (data & 0xFF));
+        out.write((byte) (data >>> 8));
+        out.write((byte) (data >>> 16));
+        out.write((byte) (data >>> 24));
+    }
+
+    public static void writeUnsignedShortLittleEndian(int data, ByteArrayOutputStream out) {
+        out.write((byte) (data & 0xFF));
+        out.write((byte) ((data >>> 8) & 0xFF));
+    }
+
+    public static void writeUnsignedMediumLittleEndian(int data, ByteArrayOutputStream out) {
+        out.write((byte) (data & 0xFF));
+        out.write((byte) ((data >>> 8) & 0xFF));
+        out.write((byte) ((data >>> 16) & 0xFF));
+    }
+
+    public static void writeBinaryCodedLengthBytes(byte[] data, ByteArrayOutputStream out) throws IOException {
+        // 1. write length byte/bytes
+        if (data.length < 252) {
+            out.write((byte) data.length);
+        } else if (data.length < (1 << 16L)) {
+            out.write((byte) 252);
+            writeUnsignedShortLittleEndian(data.length, out);
+        } else if (data.length < (1 << 24L)) {
+            out.write((byte) 253);
+            writeUnsignedMediumLittleEndian(data.length, out);
+        } else {
+            out.write((byte) 254);
+            writeUnsignedIntLittleEndian(data.length, out);
+        }
+        // 2. write real data followed length byte/bytes
+        out.write(data);
+    }
+
+    public static void writeFixedLengthBytes(byte[] data, int index, int length, ByteArrayOutputStream out) {
+        for (int i = index; i < index + length; i++) {
+            out.write(data[i]);
+        }
+    }
+
+    public static void writeFixedLengthBytesFromStart(byte[] data, int length, ByteArrayOutputStream out) {
+        writeFixedLengthBytes(data, 0, length, out);
+    }
+
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ChannelBufferHelper.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ChannelBufferHelper.java
new file mode 100644
index 00000000..2232bef0
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/ChannelBufferHelper.java
@@ -0,0 +1,62 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.io.IOException;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.IPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.PacketWithHeaderPacket;
+
+public class ChannelBufferHelper {
+
+    protected transient final Logger logger = LoggerFactory.getLogger(ChannelBufferHelper.class);
+
+    public final HeaderPacket assembleHeaderPacket(ChannelBuffer buffer) {
+        HeaderPacket header = new HeaderPacket();
+        byte[] headerBytes = new byte[MSC.HEADER_PACKET_LENGTH];
+        buffer.readBytes(headerBytes);
+        header.fromBytes(headerBytes);
+        return header;
+    }
+
+    public final PacketWithHeaderPacket assembleBodyPacketWithHeader(ChannelBuffer buffer, HeaderPacket header,
+                                                                     PacketWithHeaderPacket body) throws IOException {
+        if (body.getHeader() == null) {
+            body.setHeader(header);
+        }
+        logger.debug("body packet type:{}", body.getClass());
+        logger.debug("read body packet with packet length: {} ", header.getPacketBodyLength());
+        byte[] packetBytes = new byte[header.getPacketBodyLength()];
+
+        logger.debug("readable bytes before reading body:{}", buffer.readableBytes());
+        buffer.readBytes(packetBytes);
+        body.fromBytes(packetBytes);
+
+        logger.debug("body packet: {}", body);
+        return body;
+    }
+
+    public final ChannelBuffer createHeaderWithPacketNumberPlusOne(int bodyLength, byte packetNumber) {
+        HeaderPacket header = new HeaderPacket();
+        header.setPacketBodyLength(bodyLength);
+        header.setPacketSequenceNumber((byte) (packetNumber + 1));
+        return ChannelBuffers.wrappedBuffer(header.toBytes());
+    }
+
+    public final ChannelBuffer createHeader(int bodyLength, byte packetNumber) {
+        HeaderPacket header = new HeaderPacket();
+        header.setPacketBodyLength(bodyLength);
+        header.setPacketSequenceNumber(packetNumber);
+        return ChannelBuffers.wrappedBuffer(header.toBytes());
+    }
+
+    public final ChannelBuffer buildChannelBufferFromCommandPacket(IPacket packet) throws IOException {
+        byte[] bodyBytes = packet.toBytes();
+        ChannelBuffer header = createHeader(bodyBytes.length, (byte) 0);
+        return ChannelBuffers.wrappedBuffer(header, ChannelBuffers.wrappedBuffer(bodyBytes));
+    }
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/LengthCodedStringReader.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/LengthCodedStringReader.java
new file mode 100644
index 00000000..2877b6bc
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/LengthCodedStringReader.java
@@ -0,0 +1,43 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.io.IOException;
+
+import org.apache.commons.lang.ArrayUtils;
+
+public class LengthCodedStringReader {
+
+    public static final String CODE_PAGE_1252 = "UTF-8";
+
+    private String             encoding;
+    private int                index          = 0;      // 数组下标
+
+    public LengthCodedStringReader(String encoding, int startIndex){
+        this.encoding = encoding;
+        this.index = startIndex;
+    }
+
+    public String readLengthCodedString(byte[] data) throws IOException {
+        byte[] lengthBytes = ByteHelper.readBinaryCodedLengthBytes(data, getIndex());
+        long length = ByteHelper.readLengthCodedBinary(data, getIndex());
+        setIndex(getIndex() + lengthBytes.length);
+        if (ByteHelper.NULL_LENGTH == length) {
+            return null;
+        }
+
+        try {
+            return new String(ArrayUtils.subarray(data, getIndex(), (int) (getIndex() + length)),
+                              encoding == null ? CODE_PAGE_1252 : encoding);
+        } finally {
+            setIndex((int) (getIndex() + length));
+        }
+
+    }
+
+    public void setIndex(int index) {
+        this.index = index;
+    }
+
+    public int getIndex() {
+        return index;
+    }
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MSC.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MSC.java
new file mode 100644
index 00000000..fe46312c
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MSC.java
@@ -0,0 +1,26 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+/**
+ * MySQL Constants.

+ * constants that is used in mysql server.

+ * 
+ * @author fujohnwang
+ */
+public abstract class MSC {
+
+    public static final int  MAX_PACKET_LENGTH                  = (1 << 24);
+    public static final int  HEADER_PACKET_LENGTH_FIELD_LENGTH  = 3;
+    public static final int  HEADER_PACKET_LENGTH_FIELD_OFFSET  = 0;
+    public static final int  HEADER_PACKET_LENGTH               = 4;
+    public static final int  HEADER_PACKET_NUMBER_FIELD_LENGTH  = 1;
+
+    public static final byte NULL_TERMINATED_STRING_DELIMITER   = 0x00;
+    public static final byte DEFAULT_PROTOCOL_VERSION           = 0x0a;
+
+    public static final int  FIELD_COUNT_FIELD_LENGTH           = 1;
+
+    public static final int  EVENT_TYPE_OFFSET                  = 4;
+    public static final int  EVENT_LEN_OFFSET                   = 9;
+
+    public static final long DEFAULT_BINLOG_FILE_START_POSITION = 4;
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MySQLPasswordEncrypter.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MySQLPasswordEncrypter.java
new file mode 100644
index 00000000..85380690
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/MySQLPasswordEncrypter.java
@@ -0,0 +1,74 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class MySQLPasswordEncrypter {
+
+    public static final byte[] scramble411(byte[] pass, byte[] seed) throws NoSuchAlgorithmException {
+        MessageDigest md = MessageDigest.getInstance("SHA-1");
+        byte[] pass1 = md.digest(pass);
+        md.reset();
+        byte[] pass2 = md.digest(pass1);
+        md.reset();
+        md.update(seed);
+        byte[] pass3 = md.digest(pass2);
+        for (int i = 0; i < pass3.length; i++) {
+            pass3[i] = (byte) (pass3[i] ^ pass1[i]);
+        }
+        return pass3;
+    }
+
+    public static String scramble323(String pass, String seed) {
+        if ((pass == null) || (pass.length() == 0)) {
+            return pass;
+        }
+        byte b;
+        double d;
+        long[] pw = hash(seed);
+        long[] msg = hash(pass);
+        long max = 0x3fffffffL;
+        long seed1 = (pw[0] ^ msg[0]) % max;
+        long seed2 = (pw[1] ^ msg[1]) % max;
+        char[] chars = new char[seed.length()];
+        for (int i = 0; i < seed.length(); i++) {
+            seed1 = ((seed1 * 3) + seed2) % max;
+            seed2 = (seed1 + seed2 + 33) % max;
+            d = (double) seed1 / (double) max;
+            b = (byte) java.lang.Math.floor((d * 31) + 64);
+            chars[i] = (char) b;
+        }
+        seed1 = ((seed1 * 3) + seed2) % max;
+        seed2 = (seed1 + seed2 + 33) % max;
+        d = (double) seed1 / (double) max;
+        b = (byte) java.lang.Math.floor(d * 31);
+        for (int i = 0; i < seed.length(); i++) {
+            chars[i] ^= (char) b;
+        }
+        return new String(chars);
+    }
+
+    private static long[] hash(String src) {
+        long nr = 1345345333L;
+        long add = 7;
+        long nr2 = 0x12345671L;
+        long tmp;
+        for (int i = 0; i < src.length(); ++i) {
+            switch (src.charAt(i)) {
+                case ' ':
+                case '\t':
+                    continue;
+                default:
+                    tmp = (0xff & src.charAt(i));
+                    nr ^= ((((nr & 63) + add) * tmp) + (nr << 8));
+                    nr2 += ((nr2 << 8) ^ nr);
+                    add += tmp;
+            }
+        }
+        long[] result = new long[2];
+        result[0] = nr & 0x7fffffffL;
+        result[1] = nr2 & 0x7fffffffL;
+        return result;
+    }
+
+}
diff --git a/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java
new file mode 100644
index 00000000..7c6891ab
--- /dev/null
+++ b/driver/src/main/java/com/alibaba/otter/canal/parse/driver/mysql/utils/PacketManager.java
@@ -0,0 +1,67 @@
+package com.alibaba.otter.canal.parse.driver.mysql.utils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+
+public abstract class PacketManager {
+
+    public static HeaderPacket readHeader(SocketChannel ch, int len) throws IOException {
+        HeaderPacket header = new HeaderPacket();
+        header.fromBytes(readBytesAsBuffer(ch, len).array());
+        return header;
+    }
+
+    public static ByteBuffer readBytesAsBuffer(SocketChannel ch, int len) throws IOException {
+        ByteBuffer buffer = ByteBuffer.allocate(len);
+        while (buffer.hasRemaining()) {
+            int readNum = ch.read(buffer);
+            if (readNum == -1) {
+                throw new IOException("Unexpected End Stream");
+            }
+        }
+        return buffer;
+    }
+
+    public static byte[] readBytes(SocketChannel ch, int len) throws IOException {
+        return readBytesAsBuffer(ch, len).array();
+    }
+
+    /**
+     * Since We r using blocking IO, so we will just write once and assert the length to simplify the read operation.

+     * If the block write doesn't work as we expected, we will change this implementation as per the result.
+     * 
+     * @param ch
+     * @param len
+     * @return
+     * @throws IOException
+     */
+    public static void write(SocketChannel ch, ByteBuffer[] srcs) throws IOException {
+        @SuppressWarnings("unused")
+        long total = 0;
+        for (ByteBuffer buffer : srcs) {
+            total += buffer.remaining();
+        }
+
+        ch.write(srcs);
+        // https://github.com/alibaba/canal/issues/24
+        // 部分windows用户会出现size != total的情况，jdk为java7/openjdk，估计和java版本有关，暂时不做检查
+        // long size = ch.write(srcs);
+        // if (size != total) {
+        // throw new IOException("unexpected blocking io behavior");
+        // }
+    }
+
+    public static void write(SocketChannel ch, byte[] body) throws IOException {
+        write(ch, body, (byte) 0);
+    }
+
+    public static void write(SocketChannel ch, byte[] body, byte packetSeqNumber) throws IOException {
+        HeaderPacket header = new HeaderPacket();
+        header.setPacketBodyLength(body.length);
+        header.setPacketSequenceNumber(packetSeqNumber);
+        write(ch, new ByteBuffer[] { ByteBuffer.wrap(header.toBytes()), ByteBuffer.wrap(body) });
+    }
+}
diff --git a/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java
new file mode 100644
index 00000000..6ebcca54
--- /dev/null
+++ b/driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java
@@ -0,0 +1,54 @@
+package com.alibaba.otter.canal.parse.driver.mysql;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+
+public class MysqlConnectorTest {
+
+    @Test
+    public void testQuery() {
+
+        MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
+        try {
+            connector.connect();
+            MysqlQueryExecutor executor = new MysqlQueryExecutor(connector);
+            ResultSetPacket result = executor.query("show variables like '%char%';");
+            System.out.println(result);
+            result = executor.query("select * from test.test1");
+            System.out.println(result);
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                connector.disconnect();
+            } catch (IOException e) {
+                Assert.fail(e.getMessage());
+            }
+        }
+    }
+
+    // @Test
+    public void testUpdate() {
+
+        MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
+        try {
+            connector.connect();
+            MysqlUpdateExecutor executor = new MysqlUpdateExecutor(connector);
+            executor.update("insert into test.test2(id,name,score,text_value) values(null,'中文1',10,'中文2')");
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                connector.disconnect();
+            } catch (IOException e) {
+                Assert.fail(e.getMessage());
+            }
+        }
+    }
+}
diff --git a/example/pom.xml b/example/pom.xml
new file mode 100644
index 00000000..39791acf
--- /dev/null
+++ b/example/pom.xml
@@ -0,0 +1,123 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.example
+	jar
+	canal example module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.client
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.protocol
+			${project.version}
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+	
+	
+		
+			
+			
+				maven-jar-plugin
+				
+					
+						true
+					
+				
+			
+
+			
+				org.apache.maven.plugins
+				maven-assembly-plugin
+				
+				2.2.1
+				
+					
+						assemble
+						
+							single
+						
+						package
+					
+				
+				
+					false
+					false
+				
+			
+		
+	
+
+	
+		
+			dev
+			
+				true
+				
+					env
+					!release
+				
+			
+
+			
+				
+					
+						maven-assembly-plugin
+						
+							
+							
+								${basedir}/src/main/assembly/dev.xml
+							
+							canal-example
+							${project.build.directory}
+						
+					
+				
+			
+
+		
+
+		
+			release
+			
+				
+					env
+					release
+				
+			
+
+			
+				
+					
+						maven-assembly-plugin
+						
+							
+							
+								${basedir}/src/main/assembly/release.xml
+							
+							
+							${project.artifactId}-${project.version}
+							
+							${project.parent.build.directory}
+						
+					
+				
+			
+		
+	
+
diff --git a/example/src/main/assembly/dev.xml b/example/src/main/assembly/dev.xml
new file mode 100644
index 00000000..8f1096c5
--- /dev/null
+++ b/example/src/main/assembly/dev.xml
@@ -0,0 +1,54 @@
+
+	dist
+	
+		dir
+	
+	false
+	
+		
+			.
+			/
+			
+				README*
+			
+		
+		
+			./src/main/bin
+			bin
+			
+				**/*
+			
+			0755
+		
+		
+			./src/main/conf
+			/conf
+			
+				**/*
+			
+		
+		
+			./src/main/resources
+			/conf
+			
+				**/*
+			
+		
+		
+			target
+			logs
+			
+				**/*
+			
+		
+	
+	
+		
+			lib
+			
+				junit:junit
+			
+		
+	
+
diff --git a/example/src/main/assembly/release.xml b/example/src/main/assembly/release.xml
new file mode 100644
index 00000000..aada5f5d
--- /dev/null
+++ b/example/src/main/assembly/release.xml
@@ -0,0 +1,54 @@
+
+	dist
+	
+		tar.gz
+	
+	false
+	
+		
+			.
+			/
+			
+				README*
+			
+		
+		
+			./src/main/bin
+			bin
+			
+				**/*
+			
+			0755
+		
+		
+			./src/main/conf
+			/conf
+			
+				**/*
+			
+		
+		
+			./src/main/resources
+			/conf
+			
+				**/*
+			
+		
+		
+			target
+			logs
+			
+				**/*
+			
+		
+	
+	
+		
+			lib
+			
+				junit:junit
+			
+		
+	
+
diff --git a/example/src/main/bin/startup.bat b/example/src/main/bin/startup.bat
new file mode 100755
index 00000000..94330f09
--- /dev/null
+++ b/example/src/main/bin/startup.bat
@@ -0,0 +1,26 @@
+@echo off
+@if not "%ECHO%" == ""  echo %ECHO%
+@if "%OS%" == "Windows_NT"  setlocal
+
+set ENV_PATH=.\
+if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
+
+set conf_dir=%ENV_PATH%\..\conf
+set logback_configurationFile=%conf_dir%\logback.xml
+set client_mode=Simple
+if "%1%" != "" set client_mode=%1%
+
+set CLASSPATH=%conf_dir%
+set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
+
+set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m
+set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
+set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9199,server=y,suspend=n
+set CANAL_OPTS= -DappName=otter-canal-example -Dlogback.configurationFile="%logback_configurationFile%"
+
+set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
+
+if "%client_mode%" == "Cluster"  
+	java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.ClusterCanalClientTest
+else 
+	java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.example.SimpleCanalClientTest
diff --git a/example/src/main/bin/startup.sh b/example/src/main/bin/startup.sh
new file mode 100644
index 00000000..73b1d482
--- /dev/null
+++ b/example/src/main/bin/startup.sh
@@ -0,0 +1,96 @@
+#!/bin/bash 
+
+current_path=`pwd`
+case "`uname`" in
+    Linux)
+		bin_abs_path=$(readlink -f $(dirname $0))
+		;;
+	*)
+		bin_abs_path=`cd $(dirname $0); pwd`
+		;;
+esac
+base=${bin_abs_path}/..
+client_mode="Simple"
+logback_configurationFile=$base/conf/logback.xml
+export LANG=en_US.UTF-8
+export BASE=$base
+
+if [ -f $base/bin/canal.pid ] ; then
+	echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2
+    exit 1
+fi
+
+## set java path
+if [ -z "$JAVA" ] ; then
+  JAVA=$(which java)
+fi
+
+ALIBABA_JAVA="/usr/alibaba/java/bin/java"
+TAOBAO_JAVA="/opt/taobao/java/bin/java"
+if [ -z "$JAVA" ]; then
+  if [ -f $ALIBABA_JAVA ] ; then
+  	JAVA=$ALIBABA_JAVA
+  elif [ -f $TAOBAO_JAVA ] ; then
+  	JAVA=$TAOBAO_JAVA
+  else
+  	echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
+    exit 1
+  fi
+fi
+
+case "$#" 
+in
+0 ) 
+	;;
+1 )	
+	client_mode=$*
+	;;
+2 )	
+	if [ "$1" = "debug" ]; then
+		DEBUG_PORT=$2
+		DEBUG_SUSPEND="y"
+		JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
+	else 
+		client_mode=$1
+ 	fi;;
+* )
+	echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
+	exit;;
+esac
+
+str=`file $JAVA_HOME/bin/java | grep 64-bit`
+if [ -n "$str" ]; then
+	JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError"
+else
+	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m "
+fi
+
+JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
+CANAL_OPTS="-DappName=otter-canal-example -Dlogback.configurationFile=$logback_configurationFile"
+
+if [ -e $logback_configurationFile ]
+then 
+	
+	for i in $base/lib/*;
+		do CLASSPATH=$i:"$CLASSPATH";
+	done
+ 	CLASSPATH="$base/conf:$CLASSPATH";
+ 	
+ 	echo "cd to $bin_abs_path for workaround relative path"
+  	cd $bin_abs_path
+ 	
+	echo LOG CONFIGURATION : $logback_configurationFile
+	echo client mode : $client_mode 
+	echo CLASSPATH :$CLASSPATH
+	if [ $client_mode == "Cluster" ] ; then 
+		$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.example.ClusterCanalClientTest 1>>$base/bin/nohup.out 2>&1 &
+	else 
+		$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.example.SimpleCanalClientTest 1>>$base/bin/nohup.out 2>&1 &
+	fi
+	
+	echo $! > $base/bin/canal.pid 
+	echo "cd to $current_path for continue"
+  	cd $current_path
+else 
+	echo "client mode("$client_mode") OR log configration file($logback_configurationFile) is not exist,please create then first!"
+fi
diff --git a/example/src/main/bin/stop.sh b/example/src/main/bin/stop.sh
new file mode 100644
index 00000000..b8c997f0
--- /dev/null
+++ b/example/src/main/bin/stop.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+cygwin=false;
+case "`uname`" in
+    CYGWIN*)
+        cygwin=true
+        ;;
+esac
+
+get_pid() {	
+	STR=$1
+	PID=$2
+    if $cygwin; then
+        JAVA_CMD="$JAVA_HOME\bin\java"
+        JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
+        JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
+    else
+        if [ ! -z "$PID" ]; then
+        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+	    else 
+	        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+        fi
+    fi
+    echo $JAVA_PID;
+}
+
+base=`dirname $0`/..
+pidfile=$base/bin/canal.pid
+if [ ! -f "$pidfile" ];then
+	echo "canal is not running. exists"
+	exit
+fi
+
+pid=`cat $pidfile`
+if [ "$pid" == "" ] ; then
+	pid=`get_pid "appName=otter-canal-example"`
+fi
+
+echo -e "`hostname`: stopping canal $pid ... "
+kill $pid
+
+LOOPS=0
+while (true); 
+do 
+	gpid=`get_pid "appName=otter-canal-example" "$pid"`
+    if [ "$gpid" == "" ] ; then
+    	echo "Oook! cost:$LOOPS"
+    	`rm $pidfile`
+    	break;
+    fi
+    let LOOPS=LOOPS+1
+    sleep 1
+done
\ No newline at end of file
diff --git a/example/src/main/conf/logback.xml b/example/src/main/conf/logback.xml
new file mode 100644
index 00000000..64255e8f
--- /dev/null
+++ b/example/src/main/conf/logback.xml
@@ -0,0 +1,73 @@
+
+	
+	
+		
+			%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+			
+		
+	
+	
+	
+		
+			destination
+			canal
+		
+		
+			
+				../logs/${destination}/${destination}.log
+				
+					
+					../logs/${destination}/%d{yyyy-MM-dd}/${destination}-%d{yyyy-MM-dd}-%i.log.gz
+					
+						
+						512MB
+					
+					60
+				
+				
+					
+						%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+					
+				
+			
+		
+	
+	
+	
+		
+			destination
+			canal
+		
+		
+			
+				../logs/${destination}/entry.log
+				
+					
+					../logs/${destination}/%d{yyyy-MM-dd}/entry-%d{yyyy-MM-dd}-%i.log.gz
+					
+						512MB
+					
+					60
+				
+				
+					%msg
+				
+			
+		
+	
+	
+	  
+          
+        
+    
+	
+	
+		
+		
+	
+
\ No newline at end of file
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
new file mode 100644
index 00000000..59a378a0
--- /dev/null
+++ b/example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
@@ -0,0 +1,253 @@
+package com.alibaba.otter.canal.example;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang.SystemUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin;
+import com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd;
+import com.alibaba.otter.canal.protocol.Message;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * 测试基类
+ * 
+ * @author jianghang 2013-4-15 下午04:17:12
+ * @version 1.0.4
+ */
+public class AbstractCanalClientTest {
+
+    protected final static Logger             logger             = LoggerFactory.getLogger(AbstractCanalClientTest.class);
+    protected static final String             SEP                = SystemUtils.LINE_SEPARATOR;
+    protected static final String             DATE_FORMAT        = "yyyy-MM-dd HH:mm:ss";
+    protected volatile boolean                running            = false;
+    protected Thread.UncaughtExceptionHandler handler            = new Thread.UncaughtExceptionHandler() {
+
+                                                                     public void uncaughtException(Thread t, Throwable e) {
+                                                                         logger.error("parse events has an error", e);
+                                                                     }
+                                                                 };
+    protected Thread                          thread             = null;
+    protected CanalConnector                  connector;
+    protected static String                   context_format     = null;
+    protected static String                   row_format         = null;
+    protected static String                   transaction_format = null;
+    protected String                          destination;
+
+    static {
+        context_format = SEP + "****************************************************" + SEP;
+        context_format += "* Batch Id: [{}] ,count : [{}] , memsize : [{}] , Time : {}" + SEP;
+        context_format += "* Start : [{}] " + SEP;
+        context_format += "* End : [{}] " + SEP;
+        context_format += "****************************************************" + SEP;
+
+        row_format = SEP
+                     + "----------------> binlog[{}:{}] , name[{},{}] , eventType : {} , executeTime : {} , delay : {}ms"
+                     + SEP;
+
+        transaction_format = SEP + "================> binlog[{}:{}] , executeTime : {} , delay : {}ms" + SEP;
+
+    }
+
+    public AbstractCanalClientTest(String destination){
+        this(destination, null);
+    }
+
+    public AbstractCanalClientTest(String destination, CanalConnector connector){
+        this.destination = destination;
+        this.connector = connector;
+    }
+
+    protected void start() {
+        Assert.notNull(connector, "connector is null");
+        thread = new Thread(new Runnable() {
+
+            public void run() {
+                process();
+            }
+        });
+
+        thread.setUncaughtExceptionHandler(handler);
+        thread.start();
+        running = true;
+    }
+
+    protected void stop() {
+        if (!running) {
+            return;
+        }
+        running = false;
+        if (thread != null) {
+            try {
+                thread.join();
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+
+        MDC.remove("destination");
+    }
+
+    protected void process() {
+        int batchSize = 5 * 1024;
+        while (running) {
+            try {
+                MDC.put("destination", destination);
+                connector.connect();
+                connector.subscribe();
+                while (running) {
+                    Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
+                    long batchId = message.getId();
+                    int size = message.getEntries().size();
+                    if (batchId == -1 || size == 0) {
+                        // try {
+                        // Thread.sleep(1000);
+                        // } catch (InterruptedException e) {
+                        // }
+                    } else {
+                        printSummary(message, batchId, size);
+                        printEntry(message.getEntries());
+                    }
+
+                    connector.ack(batchId); // 提交确认
+                    // connector.rollback(batchId); // 处理失败, 回滚数据
+                }
+            } catch (Exception e) {
+                logger.error("process error!", e);
+            } finally {
+                connector.disconnect();
+                MDC.remove("destination");
+            }
+        }
+    }
+
+    private void printSummary(Message message, long batchId, int size) {
+        long memsize = 0;
+        for (Entry entry : message.getEntries()) {
+            memsize += entry.getHeader().getEventLength();
+        }
+
+        String startPosition = null;
+        String endPosition = null;
+        if (!CollectionUtils.isEmpty(message.getEntries())) {
+            startPosition = buildPositionForDump(message.getEntries().get(0));
+            endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
+        }
+
+        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+        logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
+                endPosition });
+    }
+
+    protected String buildPositionForDump(Entry entry) {
+        long time = entry.getHeader().getExecuteTime();
+        Date date = new Date(time);
+        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+        return entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":"
+               + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")";
+    }
+
+    protected void printEntry(List entrys) {
+        for (Entry entry : entrys) {
+            long executeTime = entry.getHeader().getExecuteTime();
+            long delayTime = new Date().getTime() - executeTime;
+
+            if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
+                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN) {
+                    TransactionBegin begin = null;
+                    try {
+                        begin = TransactionBegin.parseFrom(entry.getStoreValue());
+                    } catch (InvalidProtocolBufferException e) {
+                        throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
+                    }
+                    // 打印事务头信息，执行的线程id，事务耗时
+                    logger.info(transaction_format,
+                        new Object[] { entry.getHeader().getLogfileName(),
+                                String.valueOf(entry.getHeader().getLogfileOffset()),
+                                String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
+                    logger.info(" BEGIN ----> Thread id: {}", begin.getThreadId());
+                } else if (entry.getEntryType() == EntryType.TRANSACTIONEND) {
+                    TransactionEnd end = null;
+                    try {
+                        end = TransactionEnd.parseFrom(entry.getStoreValue());
+                    } catch (InvalidProtocolBufferException e) {
+                        throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
+                    }
+                    // 打印事务提交信息，事务id
+                    logger.info("----------------\n");
+                    logger.info(" END ----> transaction id: {}", end.getTransactionId());
+                    logger.info(transaction_format,
+                        new Object[] { entry.getHeader().getLogfileName(),
+                                String.valueOf(entry.getHeader().getLogfileOffset()),
+                                String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
+                }
+
+                continue;
+            }
+
+            if (entry.getEntryType() == EntryType.ROWDATA) {
+                RowChange rowChage = null;
+                try {
+                    rowChage = RowChange.parseFrom(entry.getStoreValue());
+                } catch (Exception e) {
+                    throw new RuntimeException("parse event has an error , data:" + entry.toString(), e);
+                }
+
+                EventType eventType = rowChage.getEventType();
+
+                logger.info(row_format,
+                    new Object[] { entry.getHeader().getLogfileName(),
+                            String.valueOf(entry.getHeader().getLogfileOffset()), entry.getHeader().getSchemaName(),
+                            entry.getHeader().getTableName(), eventType,
+                            String.valueOf(entry.getHeader().getExecuteTime()), String.valueOf(delayTime) });
+
+                if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
+                    logger.info(" sql ----> " + rowChage.getSql() + SEP);
+                    continue;
+                }
+
+                for (RowData rowData : rowChage.getRowDatasList()) {
+                    if (eventType == EventType.DELETE) {
+                        printColumn(rowData.getBeforeColumnsList());
+                    } else if (eventType == EventType.INSERT) {
+                        printColumn(rowData.getAfterColumnsList());
+                    } else {
+                        printColumn(rowData.getAfterColumnsList());
+                    }
+                }
+            }
+        }
+    }
+
+    protected void printColumn(List columns) {
+        for (Column column : columns) {
+            StringBuilder builder = new StringBuilder();
+            builder.append(column.getName() + " : " + column.getValue());
+            builder.append("    type=" + column.getMysqlType());
+            if (column.getUpdated()) {
+                builder.append("    update=" + column.getUpdated());
+            }
+            builder.append(SEP);
+            logger.info(builder.toString());
+        }
+    }
+
+    public void setConnector(CanalConnector connector) {
+        this.connector = connector;
+    }
+
+}
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/ClusterCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/ClusterCanalClientTest.java
new file mode 100644
index 00000000..478cc304
--- /dev/null
+++ b/example/src/main/java/com/alibaba/otter/canal/example/ClusterCanalClientTest.java
@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.example;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalConnectors;
+
+/**
+ * 集群模式的测试例子
+ * 
+ * @author jianghang 2013-4-15 下午04:19:20
+ * @version 1.0.4
+ */
+public class ClusterCanalClientTest extends AbstractCanalClientTest {
+
+    public ClusterCanalClientTest(String destination){
+        super(destination);
+    }
+
+    public static void main(String args[]) {
+        String destination = "example";
+
+        // 基于固定canal server的地址，建立链接，其中一台server发生crash，可以支持failover
+        // CanalConnector connector = CanalConnectors.newClusterConnector(
+        // Arrays.asList(new InetSocketAddress(
+        // AddressUtils.getHostIp(),
+        // 11111)),
+        // "stability_test", "", "");
+
+        // 基于zookeeper动态获取canal server的地址，建立链接，其中一台server发生crash，可以支持failover
+        CanalConnector connector = CanalConnectors.newClusterConnector("127.0.0.1:2181", destination, "", "");
+
+        final ClusterCanalClientTest clientTest = new ClusterCanalClientTest(destination);
+        clientTest.setConnector(connector);
+        clientTest.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+
+            public void run() {
+                try {
+                    logger.info("## stop the canal client");
+                    clientTest.stop();
+                } catch (Throwable e) {
+                    logger.warn("##something goes wrong when stopping canal:\n{}", ExceptionUtils.getFullStackTrace(e));
+                } finally {
+                    logger.info("## canal client is down.");
+                }
+            }
+
+        });
+    }
+}
diff --git a/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java
new file mode 100644
index 00000000..02d8ceea
--- /dev/null
+++ b/example/src/main/java/com/alibaba/otter/canal/example/SimpleCanalClientTest.java
@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.example;
+
+import java.net.InetSocketAddress;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalConnectors;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+
+/**
+ * 单机模式的测试例子
+ * 
+ * @author jianghang 2013-4-15 下午04:19:20
+ * @version 1.0.4
+ */
+public class SimpleCanalClientTest extends AbstractCanalClientTest {
+
+    public SimpleCanalClientTest(String destination){
+        super(destination);
+    }
+
+    public static void main(String args[]) {
+        // 根据ip，直接创建链接，无HA的功能
+        String destination = "example";
+        CanalConnector connector = CanalConnectors.newSingleConnector(new InetSocketAddress(AddressUtils.getHostIp(),
+                                                                                            11111), destination, "", "");
+
+        final SimpleCanalClientTest clientTest = new SimpleCanalClientTest(destination);
+        clientTest.setConnector(connector);
+        clientTest.start();
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+
+            public void run() {
+                try {
+                    logger.info("## stop the canal client");
+                    clientTest.stop();
+                } catch (Throwable e) {
+                    logger.warn("##something goes wrong when stopping canal:\n{}", ExceptionUtils.getFullStackTrace(e));
+                } finally {
+                    logger.info("## canal client is down.");
+                }
+            }
+
+        });
+    }
+
+}
diff --git a/example/src/main/resources/logback.xml b/example/src/main/resources/logback.xml
new file mode 100644
index 00000000..c6bbcd9f
--- /dev/null
+++ b/example/src/main/resources/logback.xml
@@ -0,0 +1,25 @@
+
+
+	
+	
+		
+			%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+			
+		
+	
+	
+	
+		
+			%msg
+		
+	
+	
+	  
+          
+        
+    
+	
+	
+		
+	
+
\ No newline at end of file
diff --git a/filter/pom.xml b/filter/pom.xml
new file mode 100644
index 00000000..da734e74
--- /dev/null
+++ b/filter/pom.xml
@@ -0,0 +1,39 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.filter
+	jar
+	canal sink module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.protocol
+			${project.version}
+		
+		
+			com.googlecode.aviator
+			aviator
+		
+		
+			oro
+			oro
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/CanalEventFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/CanalEventFilter.java
new file mode 100644
index 00000000..6a25ff12
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/CanalEventFilter.java
@@ -0,0 +1,13 @@
+package com.alibaba.otter.canal.filter;
+
+import com.alibaba.otter.canal.filter.exception.CanalFilterException;
+
+/**
+ * 数据过滤机制
+ * 
+ * @author jianghang 2012-7-20 下午03:51:27
+ */
+public interface CanalEventFilter {
+
+    boolean filter(T event) throws CanalFilterException;
+}
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java b/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
new file mode 100644
index 00000000..4ce36315
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.MalformedPatternException;
+import org.apache.oro.text.regex.Pattern;
+import org.apache.oro.text.regex.PatternCompiler;
+import org.apache.oro.text.regex.Perl5Compiler;
+
+import com.alibaba.otter.canal.filter.exception.CanalFilterException;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 提供{@linkplain Pattern}的lazy get处理
+ * 
+ * @author jianghang 2013-1-22 下午09:36:44
+ * @version 1.0.0
+ */
+public class PatternUtils {
+
+    private static Map patterns = new MapMaker().softValues().makeComputingMap(
+                                                                                                new Function() {
+
+                                                                                                    public Pattern apply(
+                                                                                                                         String pattern) {
+                                                                                                        try {
+                                                                                                            PatternCompiler pc = new Perl5Compiler();
+                                                                                                            return pc.compile(
+                                                                                                                              pattern,
+                                                                                                                              Perl5Compiler.CASE_INSENSITIVE_MASK
+                                                                                                                                      | Perl5Compiler.READ_ONLY_MASK
+                                                                                                                                      | Perl5Compiler.SINGLELINE_MASK);
+                                                                                                        } catch (MalformedPatternException e) {
+                                                                                                            throw new CanalFilterException(
+                                                                                                                                           e);
+                                                                                                        }
+                                                                                                    }
+                                                                                                });
+
+    public static Pattern getPattern(String pattern) {
+        return patterns.get(pattern);
+    }
+
+    public static void clear() {
+        patterns.clear();
+    }
+}
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java
new file mode 100644
index 00000000..84b21cc7
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterELFilter.java
@@ -0,0 +1,37 @@
+package com.alibaba.otter.canal.filter.aviater;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.filter.exception.CanalFilterException;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.googlecode.aviator.AviatorEvaluator;
+
+/**
+ * 基于aviater el表达式的匹配过滤
+ * 
+ * @author jianghang 2012-7-23 上午10:46:32
+ */
+public class AviaterELFilter implements CanalEventFilter {
+
+    public static final String ROOT_KEY = "entry";
+    private String             expression;
+
+    public AviaterELFilter(String expression){
+        this.expression = expression;
+    }
+
+    public boolean filter(CanalEntry.Entry entry) throws CanalFilterException {
+        if (StringUtils.isEmpty(expression)) {
+            return true;
+        }
+
+        Map env = new HashMap();
+        env.put(ROOT_KEY, entry);
+        return (Boolean) AviatorEvaluator.execute(expression, env);
+    }
+
+}
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
new file mode 100644
index 00000000..b634c40a
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterRegexFilter.java
@@ -0,0 +1,128 @@
+package com.alibaba.otter.canal.filter.aviater;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.filter.exception.CanalFilterException;
+import com.googlecode.aviator.AviatorEvaluator;
+import com.googlecode.aviator.Expression;
+
+/**
+ * 基于aviater进行tableName正则匹配的过滤算法
+ * 
+ * @author jianghang 2012-7-20 下午06:01:34
+ */
+public class AviaterRegexFilter implements CanalEventFilter {
+
+    private static final String             SPLIT             = ",";
+    private static final String             PATTERN_SPLIT     = "|";
+    private static final String             FILTER_EXPRESSION = "regex(pattern,target)";
+    private static final RegexFunction      regexFunction     = new RegexFunction();
+    private final Expression                exp               = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
+    static {
+        AviatorEvaluator.addFunction(regexFunction);
+    }
+
+    private static final Comparator COMPARATOR        = new StringComparator();
+
+    final private String                    pattern;
+    final private boolean                   defaultEmptyValue;
+
+    public AviaterRegexFilter(String pattern){
+        this(pattern, true);
+    }
+
+    public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
+        this.defaultEmptyValue = defaultEmptyValue;
+        List list = null;
+        if (StringUtils.isEmpty(pattern)) {
+            list = new ArrayList();
+        } else {
+            String[] ss = StringUtils.split(pattern, SPLIT);
+            list = Arrays.asList(ss);
+        }
+
+        // 对pattern按照从长到短的排序
+        // 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
+        // 的长度不一样
+        Collections.sort(list, COMPARATOR);
+        // 对pattern进行头尾完全匹配
+        list = completionPattern(list);
+        this.pattern = StringUtils.join(list, PATTERN_SPLIT);
+    }
+
+    public boolean filter(String filtered) throws CanalFilterException {
+        if (StringUtils.isEmpty(pattern)) {
+            return defaultEmptyValue;
+        }
+
+        if (StringUtils.isEmpty(filtered)) {
+            return defaultEmptyValue;
+        }
+
+        Map env = new HashMap();
+        env.put("pattern", pattern);
+        env.put("target", filtered.toLowerCase());
+        return (Boolean) exp.execute(env);
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
+     * 
+     * 
+     * foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
+     * 

+     * 
+     * 因此此类对正则表达式进行了从长到短的排序
+     * 
+     * @author zebin.xuzb 2012-10-22 下午2:02:26
+     * @version 1.0.0
+     */
+    private static class StringComparator implements Comparator {
+
+        @Override
+        public int compare(String str1, String str2) {
+            if (str1.length() > str2.length()) {
+                return -1;
+            } else if (str1.length() < str2.length()) {
+                return 1;
+            } else {
+                return 0;
+            }
+        }
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
+     * 
+     * 
+     * foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
+     * 

+     * 
+     * 因此此类对正则表达式进行头尾完全匹配
+     * 
+     * @author simon
+     * @version 1.0.0
+     */
+
+    private List completionPattern(List patterns) {
+        List result = new ArrayList();
+        for (String pattern : patterns) {
+            StringBuffer stringBuffer = new StringBuffer();
+            stringBuffer.append("^");
+            stringBuffer.append(pattern);
+            stringBuffer.append("$");
+            result.add(stringBuffer.toString());
+        }
+        return result;
+    }
+
+}
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java
new file mode 100644
index 00000000..ac7dab4e
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/AviaterSimpleFilter.java
@@ -0,0 +1,53 @@
+package com.alibaba.otter.canal.filter.aviater;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.filter.exception.CanalFilterException;
+import com.googlecode.aviator.AviatorEvaluator;
+import com.googlecode.aviator.Expression;
+
+/**
+ * 基于aviater进行tableName简单过滤计算，不支持正则匹配
+ * 
+ * @author jianghang 2012-7-20 下午05:53:30
+ */
+public class AviaterSimpleFilter implements CanalEventFilter {
+
+    private static final String SPLIT             = ",";
+
+    private static final String FILTER_EXPRESSION = "include(list,target)";
+
+    private final Expression    exp               = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
+
+    private final List  list;
+
+    public AviaterSimpleFilter(String filterExpression){
+        if (StringUtils.isEmpty(filterExpression)) {
+            list = new ArrayList();
+        } else {
+            String[] ss = filterExpression.toLowerCase().split(SPLIT);
+            list = Arrays.asList(ss);
+        }
+    }
+
+    public boolean filter(String filtered) throws CanalFilterException {
+        if (list.isEmpty()) {
+            return true;
+        }
+        if (StringUtils.isEmpty(filtered)) {
+            return true;
+        }
+        Map env = new HashMap();
+        env.put("list", list);
+        env.put("target", filtered.toLowerCase());
+        return (Boolean) exp.execute(env);
+    }
+
+}
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java
new file mode 100644
index 00000000..ec389416
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/aviater/RegexFunction.java
@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.filter.aviater;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.Perl5Matcher;
+
+import com.alibaba.otter.canal.filter.PatternUtils;
+import com.googlecode.aviator.runtime.function.AbstractFunction;
+import com.googlecode.aviator.runtime.function.FunctionUtils;
+import com.googlecode.aviator.runtime.type.AviatorBoolean;
+import com.googlecode.aviator.runtime.type.AviatorObject;
+
+/**
+ * 提供aviator regex的代码扩展
+ * 
+ * @author jianghang 2012-7-23 上午10:29:23
+ */
+public class RegexFunction extends AbstractFunction {
+
+    public AviatorObject call(Map env, AviatorObject arg1, AviatorObject arg2) {
+        String pattern = FunctionUtils.getStringValue(arg1, env);
+        String text = FunctionUtils.getStringValue(arg2, env);
+        Perl5Matcher matcher = new Perl5Matcher();
+        boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
+        return AviatorBoolean.valueOf(isMatch);
+    }
+
+    public String getName() {
+        return "regex";
+    }
+
+}
diff --git a/filter/src/main/java/com/alibaba/otter/canal/filter/exception/CanalFilterException.java b/filter/src/main/java/com/alibaba/otter/canal/filter/exception/CanalFilterException.java
new file mode 100644
index 00000000..cf297bd2
--- /dev/null
+++ b/filter/src/main/java/com/alibaba/otter/canal/filter/exception/CanalFilterException.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.filter.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalFilterException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalFilterException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalFilterException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalFilterException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalFilterException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalFilterException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java b/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java
new file mode 100644
index 00000000..ba345359
--- /dev/null
+++ b/filter/src/test/java/com/alibaba/otter/canal/filter/AviaterFilterTest.java
@@ -0,0 +1,108 @@
+package com.alibaba.otter.canal.filter;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.filter.aviater.AviaterELFilter;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.filter.aviater.AviaterSimpleFilter;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+
+public class AviaterFilterTest {
+
+    @Test
+    public void test_simple() {
+        AviaterSimpleFilter filter = new AviaterSimpleFilter("s1.t1,s2.t2");
+        boolean result = filter.filter("s1.t1");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("s1.t2");
+        Assert.assertEquals(false, result);
+
+        result = filter.filter("");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("s1.t1,s2.t2");
+        Assert.assertEquals(false, result);
+
+        result = filter.filter("s2.t2");
+        Assert.assertEquals(true, result);
+    }
+
+    @Test
+    public void test_regex() {
+        AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
+        boolean result = filter.filter("s1.t1");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("s1.t2");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("s12.t1");
+        Assert.assertEquals(false, result);
+
+        result = filter.filter("s2.t2");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("s3.t2");
+        Assert.assertEquals(false, result);
+
+        AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");
+
+        result = filter2.filter("s1.t1");
+        Assert.assertEquals(true, result);
+
+        result = filter2.filter("s1.t2");
+        Assert.assertEquals(true, result);
+
+        result = filter2.filter("s2.t1");
+        Assert.assertEquals(true, result);
+
+        AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");
+
+        result = filter3.filter("fooooot");
+        Assert.assertEquals(true, result);
+
+        AviaterRegexFilter filter4 = new AviaterRegexFilter(
+                                                            "otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
+        result = filter4.filter("otter1.otter_stability1");
+        Assert.assertEquals(true, result);
+    }
+
+    @Test
+    public void testDisordered() {
+        AviaterRegexFilter filter = new AviaterRegexFilter(
+                                                           "u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");
+
+        boolean result = filter.filter("u.abc");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("ab.x");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("ab.xyz1");
+        Assert.assertEquals(false, result);
+
+        result = filter.filter("abc.xyz");
+        Assert.assertEquals(true, result);
+
+        result = filter.filter("uv.xyz");
+        Assert.assertEquals(true, result);
+
+    }
+
+    @Test
+    public void test_el() {
+        AviaterELFilter filter = new AviaterELFilter("str(entry.entryType) == 'ROWDATA'");
+
+        CanalEntry.Entry.Builder entry = CanalEntry.Entry.newBuilder();
+        entry.setEntryType(CanalEntry.EntryType.ROWDATA);
+
+        boolean result = filter.filter(entry.build());
+        Assert.assertEquals(true, result);
+    }
+}
diff --git a/filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java b/filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java
new file mode 100644
index 00000000..b9cbd172
--- /dev/null
+++ b/filter/src/test/java/com/alibaba/otter/canal/filter/MutliAviaterFilterTest.java
@@ -0,0 +1,63 @@
+package com.alibaba.otter.canal.filter;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import junit.framework.Assert;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+
+public class MutliAviaterFilterTest {
+
+    @Test
+    public void test_simple() {
+        int count = 5;
+        ExecutorService executor = Executors.newFixedThreadPool(count);
+
+        final CountDownLatch countDown = new CountDownLatch(count);
+        final AtomicInteger successed = new AtomicInteger(0);
+        for (int i = 0; i < count; i++) {
+            executor.submit(new Runnable() {
+
+                public void run() {
+                    try {
+                        for (int i = 0; i < 100; i++) {
+                            doRegexTest();
+                            // try {
+                            // Thread.sleep(10);
+                            // } catch (InterruptedException e) {
+                            // }
+                        }
+
+                        successed.incrementAndGet();
+                    } finally {
+                        countDown.countDown();
+                    }
+                }
+            });
+        }
+
+        try {
+            countDown.await();
+        } catch (InterruptedException e) {
+        }
+
+        Assert.assertEquals(count, successed.get());
+        executor.shutdownNow();
+    }
+
+    private void doRegexTest() {
+        AviaterRegexFilter filter3 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|"
+                                                            + RandomStringUtils.randomAlphabetic(200));
+        boolean result = filter3.filter("otter1.otter_stability1");
+        Assert.assertEquals(true, result);
+        result = filter3.filter("otter2.otter_stability1");
+        Assert.assertEquals(true, result);
+    }
+
+}
diff --git a/instance/core/pom.xml b/instance/core/pom.xml
new file mode 100644
index 00000000..8d4e2f12
--- /dev/null
+++ b/instance/core/pom.xml
@@ -0,0 +1,35 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../../pom.xml
+	
+	com.alibaba.otter
+	canal.instance.core
+	jar
+	canal instance core module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.store
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.meta
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.parse
+			${project.version}
+		
+	
+
diff --git a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java
new file mode 100644
index 00000000..198071a0
--- /dev/null
+++ b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstance.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.instance.core;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.store.CanalEventStore;
+
+/**
+ * 代表单个canal实例，比如一个destination会独立一个实例
+ * 
+ * @author jianghang 2012-7-12 下午12:04:58
+ * @version 1.0.0
+ */
+public interface CanalInstance extends CanalLifeCycle {
+
+    public String getDestination();
+
+    public CanalEventParser getEventParser();
+
+    public CanalEventSink getEventSink();
+
+    public CanalEventStore getEventStore();
+
+    public CanalMetaManager getMetaManager();
+
+    public CanalAlarmHandler getAlarmHandler();
+
+    /**
+     * 客户端发生订阅/取消订阅行为
+     */
+    public boolean subscribeChange(ClientIdentity identity);
+}
diff --git a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceGenerator.java b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceGenerator.java
new file mode 100644
index 00000000..b7a8c146
--- /dev/null
+++ b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceGenerator.java
@@ -0,0 +1,16 @@
+package com.alibaba.otter.canal.instance.core;
+
+/**
+ * @author zebin.xuzb @ 2012-7-12
+ * @version 1.0.0
+ */
+public interface CanalInstanceGenerator {
+
+    /**
+     * 通过 destination 产生特定的 {@link CanalInstance}
+     * 
+     * @param destination
+     * @return
+     */
+    CanalInstance generate(String destination);
+}
diff --git a/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceSupport.java b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceSupport.java
new file mode 100644
index 00000000..8284c31d
--- /dev/null
+++ b/instance/core/src/main/java/com/alibaba/otter/canal/instance/core/CanalInstanceSupport.java
@@ -0,0 +1,105 @@
+package com.alibaba.otter.canal.instance.core;
+
+import java.util.List;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.ha.CanalHAController;
+import com.alibaba.otter.canal.parse.ha.HeartBeatHAController;
+import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
+import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+
+/**
+ * @author zebin.xuzb 2012-10-17 下午3:12:34
+ * @version 1.0.0
+ */
+public abstract class CanalInstanceSupport extends AbstractCanalLifeCycle {
+
+    protected void beforeStartEventParser(CanalEventParser eventParser) {
+
+        boolean isGroup = (eventParser instanceof GroupEventParser);
+        if (isGroup) {
+            // 处理group的模式
+            List eventParsers = ((GroupEventParser) eventParser).getEventParsers();
+            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
+                startEventParserInternal(singleEventParser, true);
+            }
+        } else {
+            startEventParserInternal(eventParser, false);
+        }
+    }
+
+    // around event parser
+    protected void afterStartEventParser(CanalEventParser eventParser) {
+        // noop
+    }
+
+    // around event parser
+    protected void beforeStopEventParser(CanalEventParser eventParser) {
+        // noop
+    }
+
+    protected void afterStopEventParser(CanalEventParser eventParser) {
+
+        boolean isGroup = (eventParser instanceof GroupEventParser);
+        if (isGroup) {
+            // 处理group的模式
+            List eventParsers = ((GroupEventParser) eventParser).getEventParsers();
+            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
+                stopEventParserInternal(singleEventParser);
+            }
+        } else {
+            stopEventParserInternal(eventParser);
+        }
+    }
+
+    /**
+     * 初始化单个eventParser，不需要考虑group
+     */
+    protected void startEventParserInternal(CanalEventParser eventParser, boolean isGroup) {
+        if (eventParser instanceof AbstractEventParser) {
+            AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
+            // 首先启动log position管理器
+            CanalLogPositionManager logPositionManager = abstractEventParser.getLogPositionManager();
+            if (!logPositionManager.isStart()) {
+                logPositionManager.start();
+            }
+        }
+
+        if (eventParser instanceof MysqlEventParser) {
+            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
+            CanalHAController haController = mysqlEventParser.getHaController();
+
+            if (haController instanceof HeartBeatHAController) {
+                ((HeartBeatHAController) haController).setCanalHASwitchable(mysqlEventParser);
+            }
+
+            if (!haController.isStart()) {
+                haController.start();
+            }
+
+        }
+    }
+
+    protected void stopEventParserInternal(CanalEventParser eventParser) {
+        if (eventParser instanceof AbstractEventParser) {
+            AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
+            // 首先启动log position管理器
+            CanalLogPositionManager logPositionManager = abstractEventParser.getLogPositionManager();
+            if (logPositionManager.isStart()) {
+                logPositionManager.stop();
+            }
+        }
+
+        if (eventParser instanceof MysqlEventParser) {
+            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
+            CanalHAController haController = mysqlEventParser.getHaController();
+            if (haController.isStart()) {
+                haController.stop();
+            }
+        }
+    }
+
+}
diff --git a/instance/manager/pom.xml b/instance/manager/pom.xml
new file mode 100644
index 00000000..0dbcfc4e
--- /dev/null
+++ b/instance/manager/pom.xml
@@ -0,0 +1,20 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../../pom.xml
+	
+	com.alibaba.otter
+	canal.instance.manager
+	jar
+	canal instance manager module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.instance.core
+			${project.version}
+		
+	
+
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalConfigClient.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalConfigClient.java
new file mode 100644
index 00000000..df240bf0
--- /dev/null
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalConfigClient.java
@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.instance.manager;
+
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+
+/**
+ * 对应canal的配置
+ * 
+ * @author jianghang 2012-7-4 下午03:09:17
+ * @version 1.0.0
+ */
+public class CanalConfigClient {
+
+    /**
+     * 根据对应的destinantion查询Canal信息
+     */
+    public Canal findCanal(String destination) {
+        // TODO 根据自己的业务实现
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * 根据对应的destinantion查询filter信息
+     */
+    public String findFilter(String destination) {
+        // TODO 根据自己的业务实现
+        throw new UnsupportedOperationException();
+    }
+
+}
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
new file mode 100644
index 00000000..0d9c0d32
--- /dev/null
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/CanalInstanceWithManager.java
@@ -0,0 +1,550 @@
+package com.alibaba.otter.canal.instance.manager;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
+import com.alibaba.otter.canal.common.alarm.LogAlarmHandler;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceSupport;
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.DataSourcing;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageScavengeMode;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.meta.MemoryMetaManager;
+import com.alibaba.otter.canal.meta.PeriodMixedMetaManager;
+import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.ha.CanalHAController;
+import com.alibaba.otter.canal.parse.ha.HeartBeatHAController;
+import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
+import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.LocalBinlogEventParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.index.FailbackLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MemoryLogPositionManager;
+import com.alibaba.otter.canal.parse.index.MetaLogPositionManager;
+import com.alibaba.otter.canal.parse.index.PeriodMixedLogPositionManager;
+import com.alibaba.otter.canal.parse.index.ZooKeeperLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.entry.EntryEventSink;
+import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
+import com.alibaba.otter.canal.store.AbstractCanalStoreScavenge;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import com.alibaba.otter.canal.store.model.BatchMode;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 单个canal实例，比如一个destination会独立一个实例
+ * 
+ * @author jianghang 2012-7-11 下午09:26:51
+ * @version 1.0.0
+ */
+public class CanalInstanceWithManager extends CanalInstanceSupport implements CanalInstance {
+
+    private static final Logger           logger = LoggerFactory.getLogger(CanalInstanceWithManager.class);
+    protected Long                        canalId;                                                         // 和manager交互唯一标示
+    protected String                      destination;                                                     // 队列名字
+    protected String                      filter;                                                          // 过滤表达式
+    protected CanalParameter              parameters;                                                      // 对应参数
+    protected CanalMetaManager            metaManager;                                                     // 消费信息管理器
+    protected CanalEventStore      eventStore;                                                      // 有序队列
+
+    protected CanalEventParser            eventParser;                                                     // 解析对应的数据信息
+    protected CanalEventSink> eventSink;                                                       // 链接parse和store的桥接器
+    protected CanalAlarmHandler           alarmHandler;                                                    // alarm报警机制
+    protected ZkClientx                   zkClientx;
+
+    public CanalInstanceWithManager(Canal canal){
+        this(canal, null);
+    }
+
+    public CanalInstanceWithManager(Canal canal, String filter){
+        this.parameters = canal.getCanalParameter();
+        this.canalId = canal.getId();
+        this.destination = canal.getName();
+        this.filter = filter;
+
+        logger.info("init CannalInstance for {}-{} with parameters:{}",
+            new Object[] { canalId, destination, parameters });
+        // 初始化报警机制
+        initAlarmHandler();
+        // 初始化metaManager
+        initMetaManager();
+        // 初始化eventStore
+        initEventStore();
+        // 初始化eventSink
+        initEventSink();
+        // 初始化eventParser;
+        initEventParser();
+
+        // 基础工具，需要提前start，会有先订阅再根据filter条件启动paser的需求
+        if (!alarmHandler.isStart()) {
+            alarmHandler.start();
+        }
+
+        if (!metaManager.isStart()) {
+            metaManager.start();
+        }
+        logger.info("init successful....");
+    }
+
+    public void start() {
+        super.start();
+        // 初始化metaManager
+        logger.info("start CannalInstance for {}-{} with parameters:{}", new Object[] { canalId, destination,
+                parameters });
+
+        if (!metaManager.isStart()) {
+            metaManager.start();
+        }
+
+        if (!alarmHandler.isStart()) {
+            alarmHandler.start();
+        }
+
+        if (!eventStore.isStart()) {
+            eventStore.start();
+        }
+
+        if (!eventSink.isStart()) {
+            eventSink.start();
+        }
+
+        if (!eventParser.isStart()) {
+            beforeStartEventParser(eventParser);
+            eventParser.start();
+        }
+
+        logger.info("start successful....");
+    }
+
+    public void stop() {
+        logger.info("stop CannalInstance for {}-{} ", new Object[] { canalId, destination });
+
+        if (eventParser.isStart()) {
+            eventParser.stop();
+            afterStopEventParser(eventParser);
+        }
+
+        if (eventSink.isStart()) {
+            eventSink.stop();
+        }
+
+        if (eventStore.isStart()) {
+            eventStore.stop();
+        }
+
+        if (metaManager.isStart()) {
+            metaManager.stop();
+        }
+
+        if (alarmHandler.isStart()) {
+            alarmHandler.stop();
+        }
+
+        // if (zkClientx != null) {
+        // zkClientx.close();
+        // }
+
+        super.stop();
+        logger.info("stop successful....");
+    }
+
+    public boolean subscribeChange(ClientIdentity identity) {
+        if (StringUtils.isNotEmpty(identity.getFilter())) {
+            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());
+
+            boolean isGroup = (eventParser instanceof GroupEventParser);
+            if (isGroup) {
+                // 处理group的模式
+                List eventParsers = ((GroupEventParser) eventParser).getEventParsers();
+                for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
+                    ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
+                }
+            } else {
+                ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
+            }
+
+        }
+
+        // filter的处理规则
+        // a. parser处理数据过滤处理
+        // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
+        // 后续内存版的一对多分发，可以考虑
+        return true;
+    }
+
+    protected void afterStartEventParser(CanalEventParser eventParser) {
+        super.afterStartEventParser(eventParser);
+
+        // 读取一下历史订阅的filter信息
+        List clientIdentitys = metaManager.listAllSubscribeInfo(destination);
+        for (ClientIdentity clientIdentity : clientIdentitys) {
+            subscribeChange(clientIdentity);
+        }
+    }
+
+    protected void initAlarmHandler() {
+        logger.info("init alarmHandler begin...");
+        alarmHandler = new LogAlarmHandler();
+        logger.info("init alarmHandler end! \n\t load CanalAlarmHandler:{} ", alarmHandler.getClass().getName());
+    }
+
+    protected void initMetaManager() {
+        logger.info("init metaManager begin...");
+        MetaMode mode = parameters.getMetaMode();
+        if (mode.isMemory()) {
+            metaManager = new MemoryMetaManager();
+        } else if (mode.isZookeeper()) {
+            metaManager = new ZooKeeperMetaManager();
+            ((ZooKeeperMetaManager) metaManager).setZkClientx(getZkclientx());
+        } else if (mode.isMixed()) {
+            // metaManager = new MixedMetaManager();
+            metaManager = new PeriodMixedMetaManager();// 换用优化过的mixed, at
+                                                       // 2012-09-11
+            // 设置内嵌的zk metaManager
+            ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+            zooKeeperMetaManager.setZkClientx(getZkclientx());
+            ((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
+        } else {
+            throw new CanalException("unsupport MetaMode for " + mode);
+        }
+
+        logger.info("init metaManager end! \n\t load CanalMetaManager:{} ", metaManager.getClass().getName());
+    }
+
+    protected void initEventStore() {
+        logger.info("init eventStore begin...");
+        StorageMode mode = parameters.getStorageMode();
+        if (mode.isMemory()) {
+            MemoryEventStoreWithBuffer memoryEventStore = new MemoryEventStoreWithBuffer();
+            memoryEventStore.setBufferSize(parameters.getMemoryStorageBufferSize());
+            memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
+            memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
+            memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
+            eventStore = memoryEventStore;
+        } else if (mode.isFile()) {
+            // 后续版本支持
+            throw new CanalException("unsupport MetaMode for " + mode);
+        } else if (mode.isMixed()) {
+            // 后续版本支持
+            throw new CanalException("unsupport MetaMode for " + mode);
+        } else {
+            throw new CanalException("unsupport MetaMode for " + mode);
+        }
+
+        if (eventStore instanceof AbstractCanalStoreScavenge) {
+            StorageScavengeMode scavengeMode = parameters.getStorageScavengeMode();
+            AbstractCanalStoreScavenge eventScavengeStore = (AbstractCanalStoreScavenge) eventStore;
+            eventScavengeStore.setDestination(destination);
+            eventScavengeStore.setCanalMetaManager(metaManager);
+            eventScavengeStore.setOnAck(scavengeMode.isOnAck());
+            eventScavengeStore.setOnFull(scavengeMode.isOnFull());
+            eventScavengeStore.setOnSchedule(scavengeMode.isOnSchedule());
+            if (scavengeMode.isOnSchedule()) {
+                eventScavengeStore.setScavengeSchedule(parameters.getScavengeSchdule());
+            }
+        }
+        logger.info("init eventStore end! \n\t load CanalEventStore:{}", eventStore.getClass().getName());
+    }
+
+    protected void initEventSink() {
+        logger.info("init eventSink begin...");
+
+        int groupSize = getGroupSize();
+        if (groupSize <= 1) {
+            eventSink = new EntryEventSink();
+        } else {
+            eventSink = new GroupEventSink(groupSize);
+        }
+
+        if (eventSink instanceof EntryEventSink) {
+            ((EntryEventSink) eventSink).setFilterTransactionEntry(false);
+            ((EntryEventSink) eventSink).setEventStore(getEventStore());
+        }
+        // if (StringUtils.isNotEmpty(filter)) {
+        // AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
+        // ((AbstractCanalEventSink) eventSink).setFilter(aviaterFilter);
+        // }
+        logger.info("init eventSink end! \n\t load CanalEventSink:{}", eventSink.getClass().getName());
+    }
+
+    protected void initEventParser() {
+        logger.info("init eventParser begin...");
+        SourcingType type = parameters.getSourcingType();
+
+        List> groupDbAddresses = parameters.getGroupDbAddresses();
+        if (!CollectionUtils.isEmpty(groupDbAddresses)) {
+            int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
+            List eventParsers = new ArrayList();
+            for (int i = 0; i < size; i++) {
+                List dbAddress = new ArrayList();
+                SourcingType lastType = null;
+                for (List groupDbAddress : groupDbAddresses) {
+                    if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
+                        throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
+                            lastType,
+                            groupDbAddress.get(i).getType()));
+                    }
+
+                    lastType = groupDbAddress.get(i).getType();
+                    dbAddress.add(groupDbAddress.get(i).getDbAddress());
+                }
+
+                // 初始化其中的一个分组parser
+                eventParsers.add(doInitEventParser(lastType, dbAddress));
+            }
+
+            if (eventParsers.size() > 1) { // 如果存在分组，构造分组的parser
+                GroupEventParser groupEventParser = new GroupEventParser();
+                groupEventParser.setEventParsers(eventParsers);
+                this.eventParser = groupEventParser;
+            } else {
+                this.eventParser = eventParsers.get(0);
+            }
+        } else {
+            // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
+            this.eventParser = doInitEventParser(type, new ArrayList());
+        }
+
+        logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
+    }
+
+    private CanalEventParser doInitEventParser(SourcingType type, List dbAddresses) {
+        CanalEventParser eventParser = null;
+        if (type.isMysql()) {
+            MysqlEventParser mysqlEventParser = new MysqlEventParser();
+            mysqlEventParser.setDestination(destination);
+            // 编码参数
+            mysqlEventParser.setConnectionCharset(Charset.forName(parameters.getConnectionCharset()));
+            mysqlEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
+            // 网络相关参数
+            mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
+            mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
+            mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
+            // 心跳检查参数
+            mysqlEventParser.setDetectingEnable(parameters.getDetectingEnable());
+            mysqlEventParser.setDetectingSQL(parameters.getDetectingSQL());
+            mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
+            // 数据库信息参数
+            mysqlEventParser.setSlaveId(parameters.getSlaveId());
+            if (!CollectionUtils.isEmpty(dbAddresses)) {
+                mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
+                    parameters.getDbUsername(),
+                    parameters.getDbPassword(),
+                    parameters.getDefaultDatabaseName()));
+
+                if (dbAddresses.size() > 1) {
+                    mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1),
+                        parameters.getDbUsername(),
+                        parameters.getDbPassword(),
+                        parameters.getDefaultDatabaseName()));
+                }
+            }
+
+            if (!CollectionUtils.isEmpty(parameters.getPositions())) {
+                EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
+                    EntryPosition.class);
+                // binlog位置参数
+                mysqlEventParser.setMasterPosition(masterPosition);
+
+                if (parameters.getPositions().size() > 1) {
+                    EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
+                        EntryPosition.class);
+                    mysqlEventParser.setStandbyPosition(standbyPosition);
+                }
+            }
+            mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
+            mysqlEventParser.setProfilingEnabled(false);
+            mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
+            eventParser = mysqlEventParser;
+        } else if (type.isLocalBinlog()) {
+            LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
+            localBinlogEventParser.setDestination(destination);
+            localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
+            localBinlogEventParser.setConnectionCharset(Charset.forName(parameters.getConnectionCharset()));
+            localBinlogEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
+            localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
+            localBinlogEventParser.setProfilingEnabled(false);
+            localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
+            localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
+            localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
+            // 数据库信息，反查表结构时需要
+            if (!CollectionUtils.isEmpty(dbAddresses)) {
+                localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
+                    parameters.getDbUsername(),
+                    parameters.getDbPassword(),
+                    parameters.getDefaultDatabaseName()));
+
+            }
+            eventParser = localBinlogEventParser;
+        } else if (type.isOracle()) {
+            throw new CanalException("unsupport SourcingType for " + type);
+        } else {
+            throw new CanalException("unsupport SourcingType for " + type);
+        }
+
+        // add transaction support at 2012-12-06
+        if (eventParser instanceof AbstractEventParser) {
+            AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
+            abstractEventParser.setTransactionSize(parameters.getTransactionSize());
+            abstractEventParser.setLogPositionManager(initLogPositionManager());
+            abstractEventParser.setAlarmHandler(getAlarmHandler());
+            abstractEventParser.setEventSink(getEventSink());
+
+            if (StringUtils.isNotEmpty(filter)) {
+                AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
+                abstractEventParser.setEventFilter(aviaterFilter);
+            }
+
+            // 设置黑名单
+            if (StringUtils.isNotEmpty(parameters.getBlackFilter())) {
+                AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(parameters.getBlackFilter());
+                abstractEventParser.setEventBlackFilter(aviaterFilter);
+            }
+        }
+        if (eventParser instanceof MysqlEventParser) {
+            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
+
+            // 初始化haController，绑定与eventParser的关系，haController会控制eventParser
+            CanalHAController haController = initHaController();
+            mysqlEventParser.setHaController(haController);
+        }
+        return eventParser;
+    }
+
+    protected CanalHAController initHaController() {
+        logger.info("init haController begin...");
+        HAMode haMode = parameters.getHaMode();
+        CanalHAController haController = null;
+        if (haMode.isHeartBeat()) {
+            haController = new HeartBeatHAController();
+            ((HeartBeatHAController) haController).setDetectingRetryTimes(parameters.getDetectingRetryTimes());
+            ((HeartBeatHAController) haController).setSwitchEnable(parameters.getHeartbeatHaEnable());
+        } else {
+            throw new CanalException("unsupport HAMode for " + haMode);
+        }
+        logger.info("init haController end! \n\t load CanalHAController:{}", haController.getClass().getName());
+
+        return haController;
+    }
+
+    protected CanalLogPositionManager initLogPositionManager() {
+        logger.info("init logPositionPersistManager begin...");
+        IndexMode indexMode = parameters.getIndexMode();
+        CanalLogPositionManager logPositionManager = null;
+        if (indexMode.isMemory()) {
+            logPositionManager = new MemoryLogPositionManager();
+        } else if (indexMode.isZookeeper()) {
+            logPositionManager = new ZooKeeperLogPositionManager();
+            ((ZooKeeperLogPositionManager) logPositionManager).setZkClientx(getZkclientx());
+        } else if (indexMode.isMixed()) {
+            logPositionManager = new PeriodMixedLogPositionManager();
+
+            ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager();
+            zooKeeperLogPositionManager.setZkClientx(getZkclientx());
+            ((PeriodMixedLogPositionManager) logPositionManager).setZooKeeperLogPositionManager(zooKeeperLogPositionManager);
+        } else if (indexMode.isMeta()) {
+            logPositionManager = new MetaLogPositionManager();
+            ((MetaLogPositionManager) logPositionManager).setMetaManager(metaManager);
+        } else if (indexMode.isMemoryMetaFailback()) {
+            MemoryLogPositionManager primaryLogPositionManager = new MemoryLogPositionManager();
+            MetaLogPositionManager failbackLogPositionManager = new MetaLogPositionManager();
+            failbackLogPositionManager.setMetaManager(metaManager);
+
+            logPositionManager = new FailbackLogPositionManager();
+            ((FailbackLogPositionManager) logPositionManager).setPrimary(primaryLogPositionManager);
+            ((FailbackLogPositionManager) logPositionManager).setFailback(failbackLogPositionManager);
+        } else {
+            throw new CanalException("unsupport indexMode for " + indexMode);
+        }
+
+        logger.info("init logPositionManager end! \n\t load CanalLogPositionManager:{}", logPositionManager.getClass()
+            .getName());
+
+        return logPositionManager;
+    }
+
+    protected void startEventParserInternal(CanalEventParser eventParser, boolean isGroup) {
+        if (eventParser instanceof AbstractEventParser) {
+            AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
+            abstractEventParser.setAlarmHandler(getAlarmHandler());
+        }
+
+        super.startEventParserInternal(eventParser, isGroup);
+    }
+
+    private int getGroupSize() {
+        List> groupDbAddresses = parameters.getGroupDbAddresses();
+        if (!CollectionUtils.isEmpty(groupDbAddresses)) {
+            return groupDbAddresses.get(0).size();
+        } else {
+            // 可能是基于tddl的启动
+            return 1;
+        }
+    }
+
+    private synchronized ZkClientx getZkclientx() {
+        // 做一下排序，保证相同的机器只使用同一个链接
+        List zkClusters = new ArrayList(parameters.getZkClusters());
+        Collections.sort(zkClusters);
+
+        return ZkClientx.getZkClient(StringUtils.join(zkClusters, ";"));
+    }
+
+    // =====================================
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public CanalMetaManager getMetaManager() {
+        return metaManager;
+    }
+
+    public CanalEventStore getEventStore() {
+        return eventStore;
+    }
+
+    public CanalEventParser getEventParser() {
+        return eventParser;
+    }
+
+    public CanalEventSink> getEventSink() {
+        return eventSink;
+    }
+
+    public CanalAlarmHandler getAlarmHandler() {
+        return alarmHandler;
+    }
+
+    public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
+        this.alarmHandler = alarmHandler;
+    }
+
+}
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/ManagerCanalInstanceGenerator.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/ManagerCanalInstanceGenerator.java
new file mode 100644
index 00000000..5e52de4e
--- /dev/null
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/ManagerCanalInstanceGenerator.java
@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.instance.manager;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+
+/**
+ * 基于manager生成对应的{@linkplain CanalInstance}
+ * 
+ * @author jianghang 2012-7-12 下午05:37:09
+ * @version 1.0.0
+ */
+public class ManagerCanalInstanceGenerator implements CanalInstanceGenerator {
+
+    private CanalConfigClient canalConfigClient;
+
+    public CanalInstance generate(String destination) {
+        Canal canal = canalConfigClient.findCanal(destination);
+        String filter = canalConfigClient.findFilter(destination);
+        return new CanalInstanceWithManager(canal, filter);
+    }
+
+    // ================ setter / getter ================
+
+    public void setCanalConfigClient(CanalConfigClient canalConfigClient) {
+        this.canalConfigClient = canalConfigClient;
+    }
+
+}
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/Canal.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/Canal.java
new file mode 100644
index 00000000..93ce104e
--- /dev/null
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/Canal.java
@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.instance.manager.model;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * 对应的canal模型对象
+ * 
+ * @author jianghang 2012-7-4 下午02:32:39
+ * @version 1.0.0
+ */
+public class Canal implements Serializable {
+
+    private static final long serialVersionUID = 8333284022624682754L;
+
+    private Long              id;
+    private String            name;                                   // 对应的名字
+    private String            desc;                                   // 描述
+    private CanalStatus       status;
+    private CanalParameter    canalParameter;                         // 参数定义
+    private Date              gmtCreate;                              // 创建时间
+    private Date              gmtModified;                            // 修改时间
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getDesc() {
+        return desc;
+    }
+
+    public void setDesc(String desc) {
+        this.desc = desc;
+    }
+
+    public CanalParameter getCanalParameter() {
+        return canalParameter;
+    }
+
+    public void setCanalParameter(CanalParameter canalParameter) {
+        this.canalParameter = canalParameter;
+    }
+
+    public CanalStatus getStatus() {
+        return status;
+    }
+
+    public void setStatus(CanalStatus status) {
+        this.status = status;
+    }
+
+    public Date getGmtCreate() {
+        return gmtCreate;
+    }
+
+    public void setGmtCreate(Date gmtCreate) {
+        this.gmtCreate = gmtCreate;
+    }
+
+    public Date getGmtModified() {
+        return gmtModified;
+    }
+
+    public void setGmtModified(Date gmtModified) {
+        this.gmtModified = gmtModified;
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+}
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
new file mode 100644
index 00000000..16c929ca
--- /dev/null
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalParameter.java
@@ -0,0 +1,865 @@
+package com.alibaba.otter.canal.instance.manager.model;
+
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * canal运行相关参数
+ * 
+ * @author jianghang 2012-7-4 下午02:52:52
+ * @version 1.0.0
+ */
+public class CanalParameter implements Serializable {
+
+    private static final long        serialVersionUID                   = -5893459662315430900L;
+    private Long                     canalId;
+
+    // 相关参数
+    private RunMode                  runMode                            = RunMode.EMBEDDED;          // 运行模式：嵌入式/服务式
+    private ClusterMode              clusterMode                        = ClusterMode.STANDALONE;    // 集群模式：单机/冷备/热备份
+
+    private Long                     zkClusterId;                                                    // zk集群id，为管理方便
+    private List             zkClusters;                                                     // zk集群地址
+
+    // meta相关参数
+    private MetaMode                 metaMode                           = MetaMode.MEMORY;           // meta机制
+
+    // storage存储
+    private Integer                  transactionSize                    = 1024;                      // 支持处理的transaction事务大小
+    private StorageMode              storageMode                        = StorageMode.MEMORY;        // 存储机制
+    private BatchMode                storageBatchMode                   = BatchMode.MEMSIZE;         // 基于大小返回结果
+    private Integer                  memoryStorageBufferSize            = 16 * 1024;                 // 内存存储的buffer大小
+    private Integer                  memoryStorageBufferMemUnit         = 1024;                      // 内存存储的buffer内存占用单位，默认为1kb
+    private String                   fileStorageDirectory;                                           // 文件存储的目录位置
+    private Integer                  fileStorageStoreCount;                                          // 每个文件store存储的记录数
+    private Integer                  fileStorageRollverCount;                                        // store文件的个数
+    private Integer                  fileStoragePercentThresold;                                     // 整个store存储占disk硬盘的百分比，超过百分比及时条数还未满也不写入
+    private StorageScavengeMode      storageScavengeMode                = StorageScavengeMode.ON_ACK;
+    private String                   scavengeSchdule;                                                // 调度规则
+
+    // replcation相关参数
+    private SourcingType             sourcingType                       = SourcingType.MYSQL;        // 数据来源类型
+    private String                   localBinlogDirectory;                                           // 本地localBinlog目录
+    private HAMode                   haMode                             = HAMode.HEARTBEAT;          // ha机制
+    // 网络链接参数
+    private Integer                  port                               = 11111;                     // 服务端口，独立运行时需要配置
+    private Integer                  defaultConnectionTimeoutInSeconds  = 30;                        // sotimeout
+    private Integer                  receiveBufferSize                  = 64 * 1024;
+    private Integer                  sendBufferSize                     = 64 * 1024;
+    // 编码信息
+    private Byte                     connectionCharsetNumber            = (byte) 33;
+    private String                   connectionCharset                  = "UTF-8";
+
+    // 数据库信息
+    private List  dbAddresses;                                                    // 数据库链接信息
+    private List> groupDbAddresses;                                               // 数据库链接信息，包含多组信息
+    private String                   dbUsername;                                                     // 数据库用户
+    private String                   dbPassword;                                                     // 数据库密码
+
+    // binlog链接信息
+    private IndexMode                indexMode;
+    private List             positions;                                                      // 数据库positions信息
+    private String                   defaultDatabaseName;                                            // 默认链接的数据库schmea
+    private Long                     slaveId;                                                        // 链接到mysql的slaveId
+    private Integer                  fallbackIntervalInSeconds          = 60;                        // 数据库发生切换查找时回退的时间
+
+    // 心跳检查信息
+    private Boolean                  detectingEnable                    = true;                      // 是否开启心跳语句
+    private Boolean                  heartbeatHaEnable                  = false;                     // 是否开启基于心跳检查的ha功能
+    private String                   detectingSQL;                                                   // 心跳sql
+    private Integer                  detectingIntervalInSeconds         = 3;                         // 检测频率
+    private Integer                  detectingTimeoutThresholdInSeconds = 30;                        // 心跳超时时间
+    private Integer                  detectingRetryTimes                = 3;                         // 心跳检查重试次数
+
+    // tddl/diamond 配置信息
+    private String                   app;
+    private String                   group;
+    // media配置信息
+    private String                   mediaGroup;
+    // metaq 存储配置信息
+    private String                   metaqStoreUri;
+
+    // ddl同步支持，隔离dml/ddl
+    private Boolean                  ddlIsolation                       = Boolean.FALSE;             // 是否将ddl单条返回
+    private Boolean                  filterTableError                   = Boolean.FALSE;             // 是否忽略表解析异常
+    private String                   blackFilter                        = null;                      // 匹配黑名单,忽略解析
+
+    // ================================== 兼容字段处理
+    private InetSocketAddress        masterAddress;                                                  // 主库信息
+    private String                   masterUsername;                                                 // 帐号
+    private String                   masterPassword;                                                 // 密码
+
+    private InetSocketAddress        standbyAddress;                                                 // 备库信息
+    private String                   standbyUsername;                                                // 帐号
+    private String                   standbyPassword;
+    private String                   masterLogfileName                  = null;                      // master起始位置
+    private Long                     masterLogfileOffest                = null;
+    private Long                     masterTimestamp                    = null;
+    private String                   standbyLogfileName                 = null;                      // standby起始位置
+    private Long                     standbyLogfileOffest               = null;
+    private Long                     standbyTimestamp                   = null;
+
+    public static enum RunMode {
+
+        /** 嵌入式 */
+        EMBEDDED,
+        /** 服务式 */
+        SERVICE;
+
+        public boolean isEmbedded() {
+            return this.equals(RunMode.EMBEDDED);
+        }
+
+        public boolean isService() {
+            return this.equals(RunMode.SERVICE);
+        }
+    }
+
+    public static enum ClusterMode {
+
+        /** 嵌入式 */
+        STANDALONE,
+        /** 冷备 */
+        STANDBY,
+        /** 热备 */
+        ACTIVE;
+
+        public boolean isStandalone() {
+            return this.equals(ClusterMode.STANDALONE);
+        }
+
+        public boolean isStandby() {
+            return this.equals(ClusterMode.STANDBY);
+        }
+
+        public boolean isActive() {
+            return this.equals(ClusterMode.ACTIVE);
+        }
+    }
+
+    public static enum HAMode {
+
+        /** 心跳检测 */
+        HEARTBEAT,
+        /** otter media */
+        MEDIA;
+
+        public boolean isHeartBeat() {
+            return this.equals(HAMode.HEARTBEAT);
+        }
+
+        public boolean isMedia() {
+            return this.equals(HAMode.MEDIA);
+        }
+
+    }
+
+    public static enum StorageMode {
+        /** 内存存储模式 */
+        MEMORY,
+        /** 文件存储模式 */
+        FILE,
+        /** 混合模式，内存+文件 */
+        MIXED;
+
+        public boolean isMemory() {
+            return this.equals(StorageMode.MEMORY);
+        }
+
+        public boolean isFile() {
+            return this.equals(StorageMode.FILE);
+        }
+
+        public boolean isMixed() {
+            return this.equals(StorageMode.MIXED);
+        }
+
+    }
+
+    public static enum StorageScavengeMode {
+        /** 在存储满的时候触发 */
+        ON_FULL,
+        /** 在每次有ack请求时触发 */
+        ON_ACK,
+        /** 定时触发，需要外部控制 */
+        ON_SCHEDULE,
+        /** 不做任何操作，由外部进行清理 */
+        NO_OP;
+
+        public boolean isOnFull() {
+            return this.equals(StorageScavengeMode.ON_FULL);
+        }
+
+        public boolean isOnAck() {
+            return this.equals(StorageScavengeMode.ON_ACK);
+        }
+
+        public boolean isOnSchedule() {
+            return this.equals(StorageScavengeMode.ON_SCHEDULE);
+        }
+
+        public boolean isNoop() {
+            return this.equals(StorageScavengeMode.NO_OP);
+        }
+    }
+
+    public static enum SourcingType {
+        /** mysql DB */
+        MYSQL,
+        /** localBinLog */
+        LOCALBINLOG,
+        /** oracle DB */
+        ORACLE,
+        /** 多库合并模式 */
+        GROUP;
+
+        public boolean isMysql() {
+            return this.equals(SourcingType.MYSQL);
+        }
+
+        public boolean isLocalBinlog() {
+            return this.equals(SourcingType.LOCALBINLOG);
+        }
+
+        public boolean isOracle() {
+            return this.equals(SourcingType.ORACLE);
+        }
+
+        public boolean isGroup() {
+            return this.equals(SourcingType.GROUP);
+        }
+    }
+
+    public static enum MetaMode {
+        /** 内存存储模式 */
+        MEMORY,
+        /** 文件存储模式 */
+        ZOOKEEPER,
+        /** 混合模式，内存+文件 */
+        MIXED;
+
+        public boolean isMemory() {
+            return this.equals(MetaMode.MEMORY);
+        }
+
+        public boolean isZookeeper() {
+            return this.equals(MetaMode.ZOOKEEPER);
+        }
+
+        public boolean isMixed() {
+            return this.equals(MetaMode.MIXED);
+        }
+    }
+
+    public static enum IndexMode {
+        /** 内存存储模式 */
+        MEMORY,
+        /** 文件存储模式 */
+        ZOOKEEPER,
+        /** 混合模式，内存+文件 */
+        MIXED,
+        /** 基于meta信息 */
+        META,
+        /** 基于内存+meta的failback实现 */
+        MEMORY_META_FAILBACK;
+
+        public boolean isMemory() {
+            return this.equals(IndexMode.MEMORY);
+        }
+
+        public boolean isZookeeper() {
+            return this.equals(IndexMode.ZOOKEEPER);
+        }
+
+        public boolean isMixed() {
+            return this.equals(IndexMode.MIXED);
+        }
+
+        public boolean isMeta() {
+            return this.equals(IndexMode.META);
+        }
+
+        public boolean isMemoryMetaFailback() {
+            return this.equals(IndexMode.MEMORY_META_FAILBACK);
+        }
+    }
+
+    public static enum BatchMode {
+        /** 对象数量 */
+        ITEMSIZE,
+
+        /** 内存大小 */
+        MEMSIZE;
+
+        public boolean isItemSize() {
+            return this == BatchMode.ITEMSIZE;
+        }
+
+        public boolean isMemSize() {
+            return this == BatchMode.MEMSIZE;
+        }
+    }
+
+    /**
+     * 数据来源描述
+     * 
+     * @author jianghang 2012-12-26 上午11:05:20
+     * @version 4.1.5
+     */
+    public static class DataSourcing implements Serializable {
+
+        private static final long serialVersionUID = -1770648468678085234L;
+        private SourcingType      type;
+        private InetSocketAddress dbAddress;
+
+        public DataSourcing(){
+
+        }
+
+        public DataSourcing(SourcingType type, InetSocketAddress dbAddress){
+            this.type = type;
+            this.dbAddress = dbAddress;
+        }
+
+        public SourcingType getType() {
+            return type;
+        }
+
+        public void setType(SourcingType type) {
+            this.type = type;
+        }
+
+        public InetSocketAddress getDbAddress() {
+            return dbAddress;
+        }
+
+        public void setDbAddress(InetSocketAddress dbAddress) {
+            this.dbAddress = dbAddress;
+        }
+
+    }
+
+    public Long getCanalId() {
+        return canalId;
+    }
+
+    public void setCanalId(Long canalId) {
+        this.canalId = canalId;
+    }
+
+    public RunMode getRunMode() {
+        return runMode;
+    }
+
+    public void setRunMode(RunMode runMode) {
+        this.runMode = runMode;
+    }
+
+    public ClusterMode getClusterMode() {
+        return clusterMode;
+    }
+
+    public void setClusterMode(ClusterMode clusterMode) {
+        this.clusterMode = clusterMode;
+    }
+
+    public List getZkClusters() {
+        return zkClusters;
+    }
+
+    public void setZkClusters(List zkClusters) {
+        this.zkClusters = zkClusters;
+    }
+
+    public MetaMode getMetaMode() {
+        return metaMode;
+    }
+
+    public void setMetaMode(MetaMode metaMode) {
+        this.metaMode = metaMode;
+    }
+
+    public StorageMode getStorageMode() {
+        return storageMode;
+    }
+
+    public void setStorageMode(StorageMode storageMode) {
+        this.storageMode = storageMode;
+    }
+
+    public Integer getMemoryStorageBufferSize() {
+        return memoryStorageBufferSize;
+    }
+
+    public void setMemoryStorageBufferSize(Integer memoryStorageBufferSize) {
+        this.memoryStorageBufferSize = memoryStorageBufferSize;
+    }
+
+    public String getFileStorageDirectory() {
+        return fileStorageDirectory;
+    }
+
+    public void setFileStorageDirectory(String fileStorageDirectory) {
+        this.fileStorageDirectory = fileStorageDirectory;
+    }
+
+    public Integer getFileStorageStoreCount() {
+        return fileStorageStoreCount;
+    }
+
+    public void setFileStorageStoreCount(Integer fileStorageStoreCount) {
+        this.fileStorageStoreCount = fileStorageStoreCount;
+    }
+
+    public Integer getFileStorageRollverCount() {
+        return fileStorageRollverCount;
+    }
+
+    public void setFileStorageRollverCount(Integer fileStorageRollverCount) {
+        this.fileStorageRollverCount = fileStorageRollverCount;
+    }
+
+    public Integer getFileStoragePercentThresold() {
+        return fileStoragePercentThresold;
+    }
+
+    public void setFileStoragePercentThresold(Integer fileStoragePercentThresold) {
+        this.fileStoragePercentThresold = fileStoragePercentThresold;
+    }
+
+    public SourcingType getSourcingType() {
+        return sourcingType;
+    }
+
+    public void setSourcingType(SourcingType sourcingType) {
+        this.sourcingType = sourcingType;
+    }
+
+    public String getLocalBinlogDirectory() {
+        return localBinlogDirectory;
+    }
+
+    public void setLocalBinlogDirectory(String localBinlogDirectory) {
+        this.localBinlogDirectory = localBinlogDirectory;
+    }
+
+    public HAMode getHaMode() {
+        return haMode;
+    }
+
+    public void setHaMode(HAMode haMode) {
+        this.haMode = haMode;
+    }
+
+    public Integer getPort() {
+        return port;
+    }
+
+    public void setPort(Integer port) {
+        this.port = port;
+    }
+
+    public Integer getDefaultConnectionTimeoutInSeconds() {
+        return defaultConnectionTimeoutInSeconds;
+    }
+
+    public void setDefaultConnectionTimeoutInSeconds(Integer defaultConnectionTimeoutInSeconds) {
+        this.defaultConnectionTimeoutInSeconds = defaultConnectionTimeoutInSeconds;
+    }
+
+    public Integer getReceiveBufferSize() {
+        return receiveBufferSize;
+    }
+
+    public void setReceiveBufferSize(Integer receiveBufferSize) {
+        this.receiveBufferSize = receiveBufferSize;
+    }
+
+    public Integer getSendBufferSize() {
+        return sendBufferSize;
+    }
+
+    public void setSendBufferSize(Integer sendBufferSize) {
+        this.sendBufferSize = sendBufferSize;
+    }
+
+    public Byte getConnectionCharsetNumber() {
+        return connectionCharsetNumber;
+    }
+
+    public void setConnectionCharsetNumber(Byte connectionCharsetNumber) {
+        this.connectionCharsetNumber = connectionCharsetNumber;
+    }
+
+    public String getConnectionCharset() {
+        return connectionCharset;
+    }
+
+    public void setConnectionCharset(String connectionCharset) {
+        this.connectionCharset = connectionCharset;
+    }
+
+    public IndexMode getIndexMode() {
+        return indexMode;
+    }
+
+    public void setIndexMode(IndexMode indexMode) {
+        this.indexMode = indexMode;
+    }
+
+    public String getDefaultDatabaseName() {
+        return defaultDatabaseName;
+    }
+
+    public void setDefaultDatabaseName(String defaultDatabaseName) {
+        this.defaultDatabaseName = defaultDatabaseName;
+    }
+
+    public Long getSlaveId() {
+        return slaveId;
+    }
+
+    public void setSlaveId(Long slaveId) {
+        this.slaveId = slaveId;
+    }
+
+    public Boolean getDetectingEnable() {
+        return detectingEnable;
+    }
+
+    public void setDetectingEnable(Boolean detectingEnable) {
+        this.detectingEnable = detectingEnable;
+    }
+
+    public String getDetectingSQL() {
+        return detectingSQL;
+    }
+
+    public void setDetectingSQL(String detectingSQL) {
+        this.detectingSQL = detectingSQL;
+    }
+
+    public Integer getDetectingIntervalInSeconds() {
+        return detectingIntervalInSeconds;
+    }
+
+    public void setDetectingIntervalInSeconds(Integer detectingIntervalInSeconds) {
+        this.detectingIntervalInSeconds = detectingIntervalInSeconds;
+    }
+
+    public Integer getDetectingTimeoutThresholdInSeconds() {
+        return detectingTimeoutThresholdInSeconds;
+    }
+
+    public void setDetectingTimeoutThresholdInSeconds(Integer detectingTimeoutThresholdInSeconds) {
+        this.detectingTimeoutThresholdInSeconds = detectingTimeoutThresholdInSeconds;
+    }
+
+    public Integer getDetectingRetryTimes() {
+        return detectingRetryTimes;
+    }
+
+    public void setDetectingRetryTimes(Integer detectingRetryTimes) {
+        this.detectingRetryTimes = detectingRetryTimes;
+    }
+
+    public StorageScavengeMode getStorageScavengeMode() {
+        return storageScavengeMode;
+    }
+
+    public void setStorageScavengeMode(StorageScavengeMode storageScavengeMode) {
+        this.storageScavengeMode = storageScavengeMode;
+    }
+
+    public String getScavengeSchdule() {
+        return scavengeSchdule;
+    }
+
+    public void setScavengeSchdule(String scavengeSchdule) {
+        this.scavengeSchdule = scavengeSchdule;
+    }
+
+    public String getApp() {
+        return app;
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public void setApp(String app) {
+        this.app = app;
+    }
+
+    public void setGroup(String group) {
+        this.group = group;
+    }
+
+    public String getMetaqStoreUri() {
+        return metaqStoreUri;
+    }
+
+    public void setMetaqStoreUri(String metaqStoreUri) {
+        this.metaqStoreUri = metaqStoreUri;
+    }
+
+    public Integer getTransactionSize() {
+        return transactionSize != null ? transactionSize : 1024;
+    }
+
+    public void setTransactionSize(Integer transactionSize) {
+        this.transactionSize = transactionSize;
+    }
+
+    public List getDbAddresses() {
+        if (dbAddresses == null) {
+            dbAddresses = new ArrayList();
+            if (masterAddress != null) {
+                dbAddresses.add(masterAddress);
+            }
+
+            if (standbyAddress != null) {
+                dbAddresses.add(standbyAddress);
+            }
+        }
+        return dbAddresses;
+    }
+
+    public List> getGroupDbAddresses() {
+        if (groupDbAddresses == null) {
+            groupDbAddresses = new ArrayList>();
+            if (dbAddresses != null) {
+                for (InetSocketAddress address : dbAddresses) {
+                    List groupAddresses = new ArrayList();
+                    groupAddresses.add(new DataSourcing(sourcingType, address));
+                    groupDbAddresses.add(groupAddresses);
+                }
+            } else {
+                if (masterAddress != null) {
+                    List groupAddresses = new ArrayList();
+                    groupAddresses.add(new DataSourcing(sourcingType, masterAddress));
+                    groupDbAddresses.add(groupAddresses);
+                }
+
+                if (standbyAddress != null) {
+                    List groupAddresses = new ArrayList();
+                    groupAddresses.add(new DataSourcing(sourcingType, standbyAddress));
+                    groupDbAddresses.add(groupAddresses);
+                }
+            }
+        }
+        return groupDbAddresses;
+    }
+
+    public void setGroupDbAddresses(List> groupDbAddresses) {
+        this.groupDbAddresses = groupDbAddresses;
+    }
+
+    public void setDbAddresses(List dbAddresses) {
+        this.dbAddresses = dbAddresses;
+    }
+
+    public String getDbUsername() {
+        if (dbUsername == null) {
+            dbUsername = (masterUsername != null ? masterUsername : standbyUsername);
+        }
+        return dbUsername;
+    }
+
+    public void setDbUsername(String dbUsername) {
+        this.dbUsername = dbUsername;
+    }
+
+    public String getDbPassword() {
+        if (dbPassword == null) {
+            dbPassword = (masterPassword != null ? masterPassword : standbyPassword);
+        }
+        return dbPassword;
+    }
+
+    public void setDbPassword(String dbPassword) {
+        this.dbPassword = dbPassword;
+    }
+
+    public List getPositions() {
+        if (positions == null) {
+            positions = new ArrayList();
+            String masterPosition = buildPosition(masterLogfileName, masterLogfileOffest, masterTimestamp);
+            if (masterPosition != null) {
+                positions.add(masterPosition);
+            }
+
+            String standbyPosition = buildPosition(standbyLogfileName, standbyLogfileOffest, standbyTimestamp);
+            if (standbyPosition != null) {
+                positions.add(standbyPosition);
+            }
+
+        }
+        return positions;
+    }
+
+    public void setPositions(List positions) {
+        this.positions = positions;
+    }
+
+    // ===========================兼容字段
+
+    private String buildPosition(String journalName, Long position, Long timestamp) {
+        StringBuilder masterBuilder = new StringBuilder();
+        if (StringUtils.isNotEmpty(journalName) || position != null || timestamp != null) {
+            masterBuilder.append('{');
+            if (StringUtils.isNotEmpty(journalName)) {
+                masterBuilder.append("\"journalName\":\"").append(journalName).append("\"");
+            }
+
+            if (position != null) {
+                if (masterBuilder.length() > 1) {
+                    masterBuilder.append(",");
+                }
+                masterBuilder.append("\"position\":").append(position);
+            }
+
+            if (timestamp != null) {
+                if (masterBuilder.length() > 1) {
+                    masterBuilder.append(",");
+                }
+                masterBuilder.append("\"timestamp\":").append(timestamp);
+            }
+            masterBuilder.append('}');
+            return masterBuilder.toString();
+        } else {
+            return null;
+        }
+    }
+
+    public void setMasterUsername(String masterUsername) {
+        this.masterUsername = masterUsername;
+    }
+
+    public void setMasterPassword(String masterPassword) {
+        this.masterPassword = masterPassword;
+    }
+
+    public void setStandbyAddress(InetSocketAddress standbyAddress) {
+        this.standbyAddress = standbyAddress;
+    }
+
+    public void setStandbyUsername(String standbyUsername) {
+        this.standbyUsername = standbyUsername;
+    }
+
+    public void setStandbyPassword(String standbyPassword) {
+        this.standbyPassword = standbyPassword;
+    }
+
+    public void setMasterLogfileName(String masterLogfileName) {
+        this.masterLogfileName = masterLogfileName;
+    }
+
+    public void setMasterLogfileOffest(Long masterLogfileOffest) {
+        this.masterLogfileOffest = masterLogfileOffest;
+    }
+
+    public void setMasterTimestamp(Long masterTimestamp) {
+        this.masterTimestamp = masterTimestamp;
+    }
+
+    public void setStandbyLogfileName(String standbyLogfileName) {
+        this.standbyLogfileName = standbyLogfileName;
+    }
+
+    public void setStandbyLogfileOffest(Long standbyLogfileOffest) {
+        this.standbyLogfileOffest = standbyLogfileOffest;
+    }
+
+    public void setStandbyTimestamp(Long standbyTimestamp) {
+        this.standbyTimestamp = standbyTimestamp;
+    }
+
+    public void setMasterAddress(InetSocketAddress masterAddress) {
+        this.masterAddress = masterAddress;
+    }
+
+    public Integer getFallbackIntervalInSeconds() {
+        return fallbackIntervalInSeconds == null ? 60 : fallbackIntervalInSeconds;
+    }
+
+    public void setFallbackIntervalInSeconds(Integer fallbackIntervalInSeconds) {
+        this.fallbackIntervalInSeconds = fallbackIntervalInSeconds;
+    }
+
+    public Boolean getHeartbeatHaEnable() {
+        return heartbeatHaEnable == null ? false : heartbeatHaEnable;
+    }
+
+    public void setHeartbeatHaEnable(Boolean heartbeatHaEnable) {
+        this.heartbeatHaEnable = heartbeatHaEnable;
+    }
+
+    public BatchMode getStorageBatchMode() {
+        return storageBatchMode == null ? BatchMode.MEMSIZE : storageBatchMode;
+    }
+
+    public void setStorageBatchMode(BatchMode storageBatchMode) {
+        this.storageBatchMode = storageBatchMode;
+    }
+
+    public Integer getMemoryStorageBufferMemUnit() {
+        return memoryStorageBufferMemUnit == null ? 1024 : memoryStorageBufferMemUnit;
+    }
+
+    public void setMemoryStorageBufferMemUnit(Integer memoryStorageBufferMemUnit) {
+        this.memoryStorageBufferMemUnit = memoryStorageBufferMemUnit;
+    }
+
+    public String getMediaGroup() {
+        return mediaGroup;
+    }
+
+    public void setMediaGroup(String mediaGroup) {
+        this.mediaGroup = mediaGroup;
+    }
+
+    public Long getZkClusterId() {
+        return zkClusterId;
+    }
+
+    public void setZkClusterId(Long zkClusterId) {
+        this.zkClusterId = zkClusterId;
+    }
+
+    public Boolean getDdlIsolation() {
+        return ddlIsolation == null ? false : ddlIsolation;
+    }
+
+    public void setDdlIsolation(Boolean ddlIsolation) {
+        this.ddlIsolation = ddlIsolation;
+    }
+
+    public Boolean getFilterTableError() {
+        return filterTableError == null ? false : filterTableError;
+    }
+
+    public void setFilterTableError(Boolean filterTableError) {
+        this.filterTableError = filterTableError;
+    }
+
+    public String getBlackFilter() {
+        return blackFilter;
+    }
+
+    public void setBlackFilter(String blackFilter) {
+        this.blackFilter = blackFilter;
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+}
diff --git a/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalStatus.java b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalStatus.java
new file mode 100644
index 00000000..ef2921b9
--- /dev/null
+++ b/instance/manager/src/main/java/com/alibaba/otter/canal/instance/manager/model/CanalStatus.java
@@ -0,0 +1,22 @@
+package com.alibaba.otter.canal.instance.manager.model;
+
+/**
+ * 运行状态
+ * 
+ * @author jianghang 2012-7-13 下午12:54:13
+ * @version 1.0.0
+ */
+public enum CanalStatus {
+    /** 启动 */
+    START,
+    /** 停止 */
+    STOP;
+
+    public boolean isStart() {
+        return this.equals(CanalStatus.START);
+    }
+
+    public boolean isStop() {
+        return this.equals(CanalStatus.STOP);
+    }
+}
diff --git a/instance/pom.xml b/instance/pom.xml
new file mode 100644
index 00000000..6afb7a18
--- /dev/null
+++ b/instance/pom.xml
@@ -0,0 +1,19 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.instance
+	pom
+	canal instance module for otter ${project.version}
+	
+	
+		core
+		manager
+		spring
+	
+
diff --git a/instance/spring/pom.xml b/instance/spring/pom.xml
new file mode 100644
index 00000000..64e1b22b
--- /dev/null
+++ b/instance/spring/pom.xml
@@ -0,0 +1,26 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../../pom.xml
+	
+	com.alibaba.otter
+	canal.instance.spring
+	jar
+	canal instance spring module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.instance.core
+			${project.version}
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
new file mode 100644
index 00000000..67352451
--- /dev/null
+++ b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/CanalInstanceWithSpring.java
@@ -0,0 +1,181 @@
+package com.alibaba.otter.canal.instance.spring;
+
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceSupport;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
+import com.alibaba.otter.canal.parse.inbound.group.GroupEventParser;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 基于spring容器启动canal实例，方便独立于manager启动
+ * 
+ * @author jianghang 2012-7-12 下午01:21:26
+ * @author zebin.xuzb
+ * @version 1.0.0
+ */
+public class CanalInstanceWithSpring extends CanalInstanceSupport implements CanalInstance {
+
+    private static final Logger                    logger = LoggerFactory.getLogger(CanalInstanceWithSpring.class);
+    private String                                 destination;
+    private CanalEventParser                       eventParser;
+    private CanalEventSink> eventSink;
+    private CanalEventStore                 eventStore;
+    private CanalMetaManager                       metaManager;
+    private CanalAlarmHandler                      alarmHandler;
+
+    public String getDestination() {
+        return this.destination;
+    }
+
+    public CanalEventParser getEventParser() {
+        return this.eventParser;
+    }
+
+    public CanalEventSink> getEventSink() {
+        return this.eventSink;
+    }
+
+    public CanalEventStore getEventStore() {
+        return this.eventStore;
+    }
+
+    public CanalMetaManager getMetaManager() {
+        return this.metaManager;
+    }
+
+    public CanalAlarmHandler getAlarmHandler() {
+        return alarmHandler;
+    }
+
+    public boolean subscribeChange(ClientIdentity identity) {
+        if (StringUtils.isNotEmpty(identity.getFilter())) {
+            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());
+
+            boolean isGroup = (eventParser instanceof GroupEventParser);
+            if (isGroup) {
+                // 处理group的模式
+                List eventParsers = ((GroupEventParser) eventParser).getEventParsers();
+                for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
+                    ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
+                }
+            } else {
+                ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
+            }
+
+        }
+
+        // filter的处理规则
+        // a. parser处理数据过滤处理
+        // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
+        // 后续内存版的一对多分发，可以考虑
+        return true;
+    }
+
+    protected void afterStartEventParser(CanalEventParser eventParser) {
+        super.afterStartEventParser(eventParser);
+
+        // 读取一下历史订阅的filter信息
+        List clientIdentitys = metaManager.listAllSubscribeInfo(destination);
+        for (ClientIdentity clientIdentity : clientIdentitys) {
+            subscribeChange(clientIdentity);
+        }
+    }
+
+    public void start() {
+        super.start();
+
+        logger.info("start CannalInstance for {}-{} ", new Object[] { 1, destination });
+        if (!metaManager.isStart()) {
+            metaManager.start();
+        }
+
+        if (!eventStore.isStart()) {
+            eventStore.start();
+        }
+
+        if (!eventSink.isStart()) {
+            eventSink.start();
+        }
+
+        if (!eventParser.isStart()) {
+            beforeStartEventParser(eventParser);
+            eventParser.start();
+            afterStartEventParser(eventParser);
+        }
+
+        logger.info("start successful....");
+    }
+
+    public void stop() {
+        logger.info("stop CannalInstance for {}-{} ", new Object[] { 1, destination });
+        if (eventParser.isStart()) {
+            beforeStopEventParser(eventParser);
+            eventParser.stop();
+            afterStopEventParser(eventParser);
+        }
+
+        if (eventSink.isStart()) {
+            eventSink.stop();
+        }
+
+        if (eventStore.isStart()) {
+            eventStore.stop();
+        }
+
+        if (metaManager.isStart()) {
+            metaManager.stop();
+        }
+
+        if (alarmHandler.isStart()) {
+            alarmHandler.stop();
+        }
+
+        // if (zkClientx != null) {
+        // zkClientx.close();
+        // }
+
+        super.stop();
+        logger.info("stop successful....");
+    }
+
+    // ======== setter ========
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void setEventParser(CanalEventParser eventParser) {
+        this.eventParser = eventParser;
+    }
+
+    public void setEventSink(CanalEventSink> eventSink) {
+        this.eventSink = eventSink;
+    }
+
+    public void setEventStore(CanalEventStore eventStore) {
+        this.eventStore = eventStore;
+    }
+
+    public void setMetaManager(CanalMetaManager metaManager) {
+        this.metaManager = metaManager;
+    }
+
+    public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
+        this.alarmHandler = alarmHandler;
+    }
+
+}
diff --git a/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java
new file mode 100644
index 00000000..690c084a
--- /dev/null
+++ b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/SpringCanalInstanceGenerator.java
@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.instance.spring;
+
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.beans.factory.BeanFactoryAware;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+
+/**
+ * @author zebin.xuzb @ 2012-7-12
+ * @version 1.0.0
+ */
+public class SpringCanalInstanceGenerator implements CanalInstanceGenerator, BeanFactoryAware {
+
+    private String      defaultName = "instance";
+    private BeanFactory beanFactory;
+
+    public CanalInstance generate(String destination) {
+        String beanName = destination;
+        if (!beanFactory.containsBean(beanName)) {
+            beanName = defaultName;
+        }
+
+        return (CanalInstance) beanFactory.getBean(beanName);
+    }
+
+    public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
+        this.beanFactory = beanFactory;
+    }
+
+}
diff --git a/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java
new file mode 100644
index 00000000..0c220720
--- /dev/null
+++ b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/PropertyPlaceholderConfigurer.java
@@ -0,0 +1,167 @@
+package com.alibaba.otter.canal.instance.spring.support;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.context.ResourceLoaderAware;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.ResourceLoader;
+import org.springframework.util.Assert;
+
+/**
+ * 扩展Spring的{@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer} ，增加默认值的功能。
+ * 例如：${placeholder:defaultValue}，假如placeholder的值不存在，则默认取得 defaultValue。
+ * 
+ * @author jianghang 2013-1-24 下午03:37:56
+ * @version 1.0.0
+ */
+public class PropertyPlaceholderConfigurer extends org.springframework.beans.factory.config.PropertyPlaceholderConfigurer implements ResourceLoaderAware, InitializingBean {
+
+    private static final String PLACEHOLDER_PREFIX = "${";
+    private static final String PLACEHOLDER_SUFFIX = "}";
+    private ResourceLoader      loader;
+    private String[]            locationNames;
+
+    public PropertyPlaceholderConfigurer(){
+        setIgnoreUnresolvablePlaceholders(true);
+    }
+
+    public void setResourceLoader(ResourceLoader loader) {
+        this.loader = loader;
+    }
+
+    public void setLocationNames(String[] locations) {
+        this.locationNames = locations;
+    }
+
+    public void afterPropertiesSet() throws Exception {
+        Assert.notNull(loader, "no resourceLoader");
+
+        if (locationNames != null) {
+            for (int i = 0; i < locationNames.length; i++) {
+                locationNames[i] = resolveSystemPropertyPlaceholders(locationNames[i]);
+            }
+        }
+
+        if (locationNames != null) {
+            List resources = new ArrayList(locationNames.length);
+
+            for (String location : locationNames) {
+                location = trimToNull(location);
+
+                if (location != null) {
+                    resources.add(loader.getResource(location));
+                }
+            }
+
+            super.setLocations(resources.toArray(new Resource[resources.size()]));
+        }
+    }
+
+    private String resolveSystemPropertyPlaceholders(String text) {
+        StringBuilder buf = new StringBuilder(text);
+
+        for (int startIndex = buf.indexOf(PLACEHOLDER_PREFIX); startIndex >= 0;) {
+            int endIndex = buf.indexOf(PLACEHOLDER_SUFFIX, startIndex + PLACEHOLDER_PREFIX.length());
+
+            if (endIndex != -1) {
+                String placeholder = buf.substring(startIndex + PLACEHOLDER_PREFIX.length(), endIndex);
+                int nextIndex = endIndex + PLACEHOLDER_SUFFIX.length();
+
+                try {
+                    String value = resolveSystemPropertyPlaceholder(placeholder);
+
+                    if (value != null) {
+                        buf.replace(startIndex, endIndex + PLACEHOLDER_SUFFIX.length(), value);
+                        nextIndex = startIndex + value.length();
+                    } else {
+                        System.err.println("Could not resolve placeholder '"
+                                           + placeholder
+                                           + "' in ["
+                                           + text
+                                           + "] as system property: neither system property nor environment variable found");
+                    }
+                } catch (Throwable ex) {
+                    System.err.println("Could not resolve placeholder '" + placeholder + "' in [" + text
+                                       + "] as system property: " + ex);
+                }
+
+                startIndex = buf.indexOf(PLACEHOLDER_PREFIX, nextIndex);
+            } else {
+                startIndex = -1;
+            }
+        }
+
+        return buf.toString();
+    }
+
+    private String resolveSystemPropertyPlaceholder(String placeholder) {
+        DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
+        String value = System.getProperty(dp.placeholder);
+
+        if (value == null) {
+            value = System.getenv(dp.placeholder);
+        }
+
+        if (value == null) {
+            value = dp.defaultValue;
+        }
+
+        return value;
+    }
+
+    @Override
+    protected String resolvePlaceholder(String placeholder, Properties props, int systemPropertiesMode) {
+        DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
+        String value = super.resolvePlaceholder(dp.placeholder, props, systemPropertiesMode);
+
+        if (value == null) {
+            value = dp.defaultValue;
+        }
+
+        return trimToEmpty(value);
+    }
+
+    private static class DefaultablePlaceholder {
+
+        private final String defaultValue;
+        private final String placeholder;
+
+        public DefaultablePlaceholder(String placeholder){
+            int commaIndex = placeholder.indexOf(":");
+            String defaultValue = null;
+
+            if (commaIndex >= 0) {
+                defaultValue = trimToEmpty(placeholder.substring(commaIndex + 1));
+                placeholder = trimToEmpty(placeholder.substring(0, commaIndex));
+            }
+
+            this.placeholder = placeholder;
+            this.defaultValue = defaultValue;
+        }
+    }
+
+    private String trimToNull(String str) {
+        if (str == null) {
+            return null;
+        }
+
+        String result = str.trim();
+
+        if (result == null || result.length() == 0) {
+            return null;
+        }
+
+        return result;
+    }
+
+    public static String trimToEmpty(String str) {
+        if (str == null) {
+            return "";
+        }
+
+        return str.trim();
+    }
+}
diff --git a/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java
new file mode 100644
index 00000000..878c01a4
--- /dev/null
+++ b/instance/spring/src/main/java/com/alibaba/otter/canal/instance/spring/support/SocketAddressEditor.java
@@ -0,0 +1,28 @@
+package com.alibaba.otter.canal.instance.spring.support;
+
+import java.beans.PropertyEditorSupport;
+import java.net.InetSocketAddress;
+
+import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.PropertyEditorRegistrar;
+import org.springframework.beans.PropertyEditorRegistry;
+
+public class SocketAddressEditor extends PropertyEditorSupport implements PropertyEditorRegistrar {
+
+    public void registerCustomEditors(PropertyEditorRegistry registry) {
+        registry.registerCustomEditor(InetSocketAddress.class, this);
+    }
+
+    public void setAsText(String text) throws IllegalArgumentException {
+        String[] addresses = StringUtils.split(text, ":");
+        if (addresses.length > 0) {
+            if (addresses.length != 2) {
+                throw new RuntimeException("address[" + text + "] is illegal, eg.127.0.0.1:3306");
+            } else {
+                setValue(new InetSocketAddress(addresses[0], Integer.valueOf(addresses[1])));
+            }
+        } else {
+            setValue(null);
+        }
+    }
+}
diff --git a/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/DefaultSpringInstanceTest.java b/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/DefaultSpringInstanceTest.java
new file mode 100644
index 00000000..37b76c5b
--- /dev/null
+++ b/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/DefaultSpringInstanceTest.java
@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.instance.spring.integrated;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.AbstractApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+
+/**
+ * @author zebin.xuzb @ 2012-7-13
+ * @version 1.0.0
+ */
+public class DefaultSpringInstanceTest {
+
+    private ApplicationContext context;
+
+    @Before
+    public void start() {
+        System.setProperty("canal.instance.destination", "retl");
+        context = new ClassPathXmlApplicationContext(new String[] { "spring/default-instance.xml" });
+    }
+
+    @After
+    public void close() {
+        if (context != null && context instanceof AbstractApplicationContext) {
+            ((AbstractApplicationContext) context).close();
+        }
+    }
+
+    @Test
+    public void testInstance() {
+        CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
+        CanalInstance canalInstance = generator.generate("instance");
+        Assert.notNull(canalInstance);
+
+        canalInstance.start();
+        try {
+            Thread.sleep(10 * 1000);
+        } catch (InterruptedException e) {
+        }
+        canalInstance.stop();
+    }
+}
diff --git a/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/GroupSpringInstanceTest.java b/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/GroupSpringInstanceTest.java
new file mode 100644
index 00000000..d7229978
--- /dev/null
+++ b/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/GroupSpringInstanceTest.java
@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.instance.spring.integrated;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.AbstractApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+
+/**
+ * @author zebin.xuzb @ 2012-7-13
+ * @version 1.0.0
+ */
+public class GroupSpringInstanceTest {
+
+    private ApplicationContext context;
+
+    @Before
+    public void start() {
+        System.setProperty("canal.instance.destination", "retl");
+        context = new ClassPathXmlApplicationContext(new String[] { "spring/group-instance.xml" });
+    }
+
+    @After
+    public void close() {
+        if (context != null && context instanceof AbstractApplicationContext) {
+            ((AbstractApplicationContext) context).close();
+        }
+    }
+
+    @Test
+    public void testInstance() {
+        CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
+        CanalInstance canalInstance = generator.generate("instance");
+        Assert.notNull(canalInstance);
+
+        canalInstance.start();
+        try {
+            Thread.sleep(10 * 1000);
+        } catch (InterruptedException e) {
+        }
+        canalInstance.stop();
+    }
+}
diff --git a/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/MemorySpringInstanceTest.java b/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/MemorySpringInstanceTest.java
new file mode 100644
index 00000000..2e900cd9
--- /dev/null
+++ b/instance/spring/src/test/java/com/alibaba/otter/canal/instance/spring/integrated/MemorySpringInstanceTest.java
@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.instance.spring.integrated;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.AbstractApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+
+/**
+ * @author zebin.xuzb @ 2012-7-13
+ * @version 1.0.0
+ */
+public class MemorySpringInstanceTest {
+
+    private ApplicationContext context;
+
+    @Before
+    public void start() {
+        System.setProperty("canal.instance.destination", "retl");
+        context = new ClassPathXmlApplicationContext(new String[] { "spring/memory-instance.xml" });
+    }
+
+    @After
+    public void close() {
+        if (context != null && context instanceof AbstractApplicationContext) {
+            ((AbstractApplicationContext) context).close();
+        }
+    }
+
+    @Test
+    public void testInstance() {
+        CanalInstanceGenerator generator = (CanalInstanceGenerator) context.getBean("canalInstanceGenerator");
+        CanalInstance canalInstance = generator.generate("instance");
+        Assert.notNull(canalInstance);
+
+        canalInstance.start();
+        try {
+            Thread.sleep(10 * 1000);
+        } catch (InterruptedException e) {
+        }
+        canalInstance.stop();
+    }
+}
diff --git a/instance/spring/src/test/resources/canal.properties b/instance/spring/src/test/resources/canal.properties
new file mode 100644
index 00000000..1887781d
--- /dev/null
+++ b/instance/spring/src/test/resources/canal.properties
@@ -0,0 +1,54 @@
+#################################################
+######### 		common argument		############# 
+#################################################
+canal.id= 1
+canal.ip=
+canal.port= 11111
+canal.zkServers=
+# flush data to zk
+canal.zookeeper.flush.period = 1000
+## memory store RingBuffer size, should be Math.pow(2,n)
+canal.instance.memory.buffer.size = 16384
+## memory store RingBuffer used memory unit size , default 1kb
+canal.instance.memory.buffer.memunit = 1024 
+## meory store gets mode used MEMSIZE or ITEMSIZE
+canal.instance.memory.batch.mode = MEMSIZE
+
+## detecing config
+canal.instance.detecting.enable = false
+canal.instance.detecting.sql = insert into retl.xdual values(1,now()) on duplicate key update x=now()
+canal.instance.detecting.interval.time = 3
+canal.instance.detecting.retry.threshold = 3
+canal.instance.detecting.heartbeatHaEnable = false
+
+# support maximum transaction size, more than the size of the transaction will be cut into multiple transactions delivery
+canal.instance.transaction.size =  1024
+# mysql fallback connected to new master should fallback times
+canal.instance.fallbackIntervalInSeconds = 60
+
+# network config
+canal.instance.network.receiveBufferSize = 16384
+canal.instance.network.sendBufferSize = 16384
+canal.instance.network.soTimeout = 30
+
+# binlog filter config
+canal.instance.filter.query.dcl = false
+canal.instance.filter.query.dml = false
+
+#################################################
+######### 		destinations		############# 
+#################################################
+canal.destinations= example
+# conf root dir
+canal.conf.dir = ../conf
+# auto scan instance dir add/remove and start/stop instance
+canal.auto.scan = true
+canal.auto.scan.interval = 5
+# as far as possible to stop canal instance where client disconnect
+canal.stopInstanceAsPossible = true
+
+canal.instance.global.mode = spring 
+canal.instance.global.lazy = false
+#canal.instance.global.manager.address = 127.0.0.1:1099
+canal.instance.global.spring.xml = classpath:spring/memory-instance.xml
+#canal.instance.global.spring.xml = classpath:spring/default-instance.xml
\ No newline at end of file
diff --git a/instance/spring/src/test/resources/retl/instance.properties b/instance/spring/src/test/resources/retl/instance.properties
new file mode 100644
index 00000000..62ba5c4c
--- /dev/null
+++ b/instance/spring/src/test/resources/retl/instance.properties
@@ -0,0 +1,37 @@
+canal.instance.detecting.enable = true
+
+#################################################
+## mysql serverId
+canal.instance.mysql.slaveId = 1234
+
+# position info
+canal.instance.master.address = 127.0.0.1:3306
+canal.instance.master.journal.name = 
+canal.instance.master.position = 
+canal.instance.master.timestamp = 
+
+canal.instance.master1.address = 127.0.0.1:3306
+canal.instance.master1.journal.name = 
+canal.instance.master1.position = 
+canal.instance.master1.timestamp = 
+
+canal.instance.master2.address = 127.0.0.1:3306
+canal.instance.master2.journal.name = 
+canal.instance.master2.position = 
+canal.instance.master2.timestamp = 
+
+#canal.instance.standby.address = 
+#canal.instance.standby.journal.name =
+#canal.instance.standby.position = 
+#canal.instance.standby.timestamp = 
+
+# username/password
+canal.instance.dbUsername = xxxxx
+canal.instance.dbPassword = xxxxx
+canal.instance.defaultDatabaseName =
+canal.instance.connectionCharset = UTF-8
+
+# table regex
+canal.instance.filter.regex = .*\\..*
+
+#################################################
\ No newline at end of file
diff --git a/instance/spring/src/test/resources/spring/default-instance.xml b/instance/spring/src/test/resources/spring/default-instance.xml
new file mode 100644
index 00000000..0e035319
--- /dev/null
+++ b/instance/spring/src/test/resources/spring/default-instance.xml
@@ -0,0 +1,173 @@
+
+
+
+	
+	
+		
+		
+		
+			
+				classpath:canal.properties
+				classpath:${canal.instance.destination:}/instance.properties
+			
+		
+	
+	
+	
+	 
+		
+			
+				
+			
+		
+	
+	
+	
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+	
+	
+	
+	
+	
+	
+		
+		
+		
+			
+				${canal.zkServers:127.0.0.1:2181}
+			
+		
+	
+	
+	
+		
+			
+				
+			
+		
+		
+	
+	
+	
+		
+		
+		
+	
+	
+	
+		
+	
+
+	
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+			
+		
+		
+		
+		
+		
+		
+			
+				
+			
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+	
+		
+		
+			
+				
+					
+				
+				
+					
+						
+					
+				
+			
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+		
+		
+			
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+			
+		
+		
+		
+	
+
\ No newline at end of file
diff --git a/instance/spring/src/test/resources/spring/file-instance.xml b/instance/spring/src/test/resources/spring/file-instance.xml
new file mode 100644
index 00000000..1acd7050
--- /dev/null
+++ b/instance/spring/src/test/resources/spring/file-instance.xml
@@ -0,0 +1,159 @@
+
+
+
+	
+	
+		
+		
+		
+			
+				classpath:canal.properties
+				classpath:${canal.instance.destination:}/instance.properties
+			
+		
+	
+	
+	
+	 
+		
+			
+				
+			
+		
+	
+	
+	
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+	
+	
+	
+	
+	
+	
+		
+		
+	
+	
+	
+		
+		
+		
+	
+	
+	
+		
+	
+
+	
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+			
+		
+		
+		
+		
+		
+		
+			
+				
+			
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+	
+		
+		
+			
+				
+					
+				
+				
+					
+						
+					
+				
+			
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+		
+		
+			
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+			
+		
+		
+		
+	
+
\ No newline at end of file
diff --git a/instance/spring/src/test/resources/spring/group-instance.xml b/instance/spring/src/test/resources/spring/group-instance.xml
new file mode 100644
index 00000000..763ec12f
--- /dev/null
+++ b/instance/spring/src/test/resources/spring/group-instance.xml
@@ -0,0 +1,236 @@
+
+
+	
+	
+	
+		
+		
+		
+			
+				classpath:canal.properties
+				classpath:${canal.instance.destination:}/instance.properties
+			
+		
+	
+	
+	
+	 
+		
+			
+				
+			
+		
+	
+
+	
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+	
+	
+	
+	
+	
+	
+	
+	
+		
+		
+		
+	
+	
+	
+		
+	
+	
+	
+		
+			
+				
+				
+			
+		
+	
+
+	
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+			
+		
+		
+		
+		
+		
+		
+			
+				
+			
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+	
+		
+		
+			
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+		
+		
+			
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+			
+		
+		
+		
+	
+	
+	
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+			
+		
+		
+		
+		
+		
+		
+			
+				
+			
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+	
+		
+		
+			
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+		
+		
+			
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+			
+		
+		
+		
+	
+
\ No newline at end of file
diff --git a/instance/spring/src/test/resources/spring/memory-instance.xml b/instance/spring/src/test/resources/spring/memory-instance.xml
new file mode 100644
index 00000000..0f58e964
--- /dev/null
+++ b/instance/spring/src/test/resources/spring/memory-instance.xml
@@ -0,0 +1,147 @@
+
+
+	
+	
+	
+		
+		
+		
+			
+				classpath:canal.properties
+				classpath:${canal.instance.destination:}/instance.properties
+			
+		
+	
+	
+	
+	 
+		
+			
+				
+			
+		
+	
+
+	
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+		
+			
+		
+	
+	
+	
+	
+	
+	
+	
+	
+		
+		
+		
+	
+	
+	
+		
+	
+
+	
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+			
+		
+		
+		
+		
+		
+		
+			
+				
+			
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+		
+	
+		
+		
+			
+		
+		
+		
+		
+		
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+				
+			
+		
+		
+		
+		
+			
+				
+				
+				
+			
+		
+		
+			
+				
+				
+				
+			
+		
+		
+		
+	
+
\ No newline at end of file
diff --git a/meta/pom.xml b/meta/pom.xml
new file mode 100644
index 00000000..14668118
--- /dev/null
+++ b/meta/pom.xml
@@ -0,0 +1,31 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.meta
+	jar
+	canal meta module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.protocol
+			${project.version}
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/CanalMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/CanalMetaManager.java
new file mode 100644
index 00000000..534e6d4a
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/CanalMetaManager.java
@@ -0,0 +1,93 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+/**
+ * meta信息管理器
+ * 
+ * @author jianghang 2012-6-14 下午09:28:48
+ * @author zebin.xuzb
+ * @version 1.0.0
+ */
+public interface CanalMetaManager extends CanalLifeCycle {
+
+    /**
+     * 增加一个 client订阅 

+     * 如果 client已经存在，则不做任何修改
+     */
+    void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 判断是否订阅
+     */
+    boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 取消client订阅
+     */
+    void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 获取cuosr游标
+     */
+    Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 更新cuosr游标
+     */
+    void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException;
+
+    /**
+     * 根据指定的destination列出当前所有的clientIdentity信息
+     */
+    List listAllSubscribeInfo(String destination) throws CanalMetaManagerException;
+
+    /**
+     * 获得该client最新的一个位置
+     */
+    PositionRange getFirstBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 获得该clientId最新的一个位置
+     */
+    PositionRange getLastestBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 为 client 产生一个唯一、递增的id
+     */
+    Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException;
+
+    /**
+     * 指定batchId，插入batch数据
+     */
+    void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
+                                                                                           throws CanalMetaManagerException;
+
+    /**
+     * 根据唯一messageId，查找对应的数据起始信息
+     */
+    PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException;
+
+    /**
+     * 对一个batch的确认
+     */
+    PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException;
+
+    /**
+     * 查询当前的所有batch信息
+     */
+    Map listAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+    /**
+     * 清除对应的batch信息
+     */
+    void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException;
+
+}
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
new file mode 100644
index 00000000..f7062c53
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
@@ -0,0 +1,375 @@
+package com.alibaba.otter.canal.meta;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于文件刷新的metaManager实现
+ * 
+ * 
+ * 策略：
+ * 1. 先写内存，然后定时刷新数据到File
+ * 2. 数据采取overwrite模式(只保留最后一次)，通过logger实施append模式(记录历史版本)
+ * 

+ * 
+ * @author jianghang 2013-4-15 下午05:55:57
+ * @version 1.0.4
+ */
+public class FileMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
+
+    private static final Logger      logger       = LoggerFactory.getLogger(FileMixedMetaManager.class);
+    private static final Charset     charset      = Charset.forName("UTF-8");
+    private File                     dataDir;
+    private String                   dataFileName = "meta.dat";
+    private Map        dataFileCaches;
+    private ScheduledExecutorService executor;
+    @SuppressWarnings("serial")
+    private final Position           nullCursor   = new Position() {
+                                                  };
+    private long                     period       = 1000;                                               // 单位ms
+    private Set      updateCursorTasks;
+
+    public void start() {
+        super.start();
+        Assert.notNull(dataDir);
+        if (!dataDir.exists()) {
+            try {
+                FileUtils.forceMkdir(dataDir);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+
+        if (!dataDir.canRead() || !dataDir.canWrite()) {
+            throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
+        }
+
+        dataFileCaches = new MapMaker().makeComputingMap(new Function() {
+
+            public File apply(String destination) {
+                return getDataFile(destination);
+            }
+        });
+
+        executor = Executors.newScheduledThreadPool(1);
+        destinations = new MapMaker().makeComputingMap(new Function>() {
+
+            public List apply(String destination) {
+                return loadClientIdentity(destination);
+            }
+        });
+
+        cursors = new MapMaker().makeComputingMap(new Function() {
+
+            public Position apply(ClientIdentity clientIdentity) {
+                Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
+                if (position == null) {
+                    return nullCursor; // 返回一个空对象标识，避免出现异常
+                } else {
+                    return position;
+                }
+            }
+        });
+
+        updateCursorTasks = Collections.synchronizedSet(new HashSet());
+
+        // 启动定时工作任务
+        executor.scheduleAtFixedRate(new Runnable() {
+
+            public void run() {
+                List tasks = new ArrayList(updateCursorTasks);
+                for (ClientIdentity clientIdentity : tasks) {
+                    MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
+                    try {
+                        // 定时将内存中的最新值刷到file中，多次变更只刷一次
+                        if (logger.isInfoEnabled()) {
+                            LogPosition cursor = (LogPosition) getCursor(clientIdentity);
+                            logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] {
+                                    clientIdentity.getClientId(), cursor.getPostion().getJournalName(),
+                                    cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(),
+                                    cursor.getIdentity().getSourceAddress().toString() });
+                        }
+                        flushDataToFile(clientIdentity.getDestination());
+                        updateCursorTasks.remove(clientIdentity);
+                    } catch (Throwable e) {
+                        // ignore
+                        logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
+                    }
+                }
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    public void stop() {
+        super.stop();
+
+        flushDataToFile();// 刷新数据
+        executor.shutdownNow();
+        destinations.clear();
+        batches.clear();
+    }
+
+    public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.subscribe(clientIdentity);
+
+        // 订阅信息频率发生比较低，不需要做定时merge处理
+        executor.submit(new Runnable() {
+
+            public void run() {
+                flushDataToFile(clientIdentity.getDestination());
+            }
+        });
+    }
+
+    public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.unsubscribe(clientIdentity);
+
+        // 订阅信息频率发生比较低，不需要做定时merge处理
+        executor.submit(new Runnable() {
+
+            public void run() {
+                flushDataToFile(clientIdentity.getDestination());
+            }
+        });
+    }
+
+    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
+        updateCursorTasks.add(clientIdentity);// 添加到任务队列中进行触发
+        super.updateCursor(clientIdentity, position);
+    }
+
+    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        Position position = super.getCursor(clientIdentity);
+        if (position == nullCursor) {
+            return null;
+        } else {
+            return position;
+        }
+    }
+
+    // ============================ helper method ======================
+
+    private File getDataFile(String destination) {
+        File destinationMetaDir = new File(dataDir, destination);
+        if (!destinationMetaDir.exists()) {
+            try {
+                FileUtils.forceMkdir(destinationMetaDir);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+
+        return new File(destinationMetaDir, dataFileName);
+    }
+
+    private FileMetaInstanceData loadDataFromFile(File dataFile) {
+        try {
+            if (!dataFile.exists()) {
+                return null;
+            }
+
+            String json = FileUtils.readFileToString(dataFile, charset.name());
+            return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
+        } catch (IOException e) {
+            throw new CanalMetaManagerException(e);
+        }
+    }
+
+    private void flushDataToFile() {
+        for (String destination : destinations.keySet()) {
+            flushDataToFile(destination);
+        }
+    }
+
+    private void flushDataToFile(String destination) {
+        flushDataToFile(destination, dataFileCaches.get(destination));
+    }
+
+    private void flushDataToFile(String destination, File dataFile) {
+        FileMetaInstanceData data = new FileMetaInstanceData();
+        if (destinations.containsKey(destination)) {
+            synchronized (destination.intern()) { // 基于destination控制一下并发更新
+                data.setDestination(destination);
+
+                List clientDatas = Lists.newArrayList();
+                List clientIdentitys = destinations.get(destination);
+                for (ClientIdentity clientIdentity : clientIdentitys) {
+                    FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
+                    clientData.setClientIdentity(clientIdentity);
+                    Position position = cursors.get(clientIdentity);
+                    if (position != null && position != nullCursor) {
+                        clientData.setCursor((LogPosition) position);
+                    }
+
+                    clientDatas.add(clientData);
+                }
+
+                data.setClientDatas(clientDatas);
+            }
+
+            String json = JsonUtils.marshalToString(data);
+            try {
+                FileUtils.writeStringToFile(dataFile, json);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+    }
+
+    private List loadClientIdentity(String destination) {
+        List result = Lists.newArrayList();
+
+        FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
+        if (data == null) {
+            return result;
+        }
+
+        List clientDatas = data.getClientDatas();
+        if (clientDatas == null) {
+            return result;
+        }
+
+        for (FileMetaClientIdentityData clientData : clientDatas) {
+            if (clientData.getClientIdentity().getDestination().equals(destination)) {
+                result.add(clientData.getClientIdentity());
+            }
+        }
+
+        return result;
+    }
+
+    private Position loadCursor(String destination, ClientIdentity clientIdentity) {
+        FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
+        if (data == null) {
+            return null;
+        }
+
+        List clientDatas = data.getClientDatas();
+        if (clientDatas == null) {
+            return null;
+        }
+
+        for (FileMetaClientIdentityData clientData : clientDatas) {
+            if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
+                return clientData.getCursor();
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * 描述一个clientIdentity对应的数据对象
+     * 
+     * @author jianghang 2013-4-15 下午06:19:40
+     * @version 1.0.4
+     */
+    public static class FileMetaClientIdentityData {
+
+        private ClientIdentity clientIdentity;
+        private LogPosition    cursor;
+
+        public FileMetaClientIdentityData(){
+
+        }
+
+        public FileMetaClientIdentityData(ClientIdentity clientIdentity, MemoryClientIdentityBatch batch,
+                                          LogPosition cursor){
+            this.clientIdentity = clientIdentity;
+            this.cursor = cursor;
+        }
+
+        public ClientIdentity getClientIdentity() {
+            return clientIdentity;
+        }
+
+        public void setClientIdentity(ClientIdentity clientIdentity) {
+            this.clientIdentity = clientIdentity;
+        }
+
+        public Position getCursor() {
+            return cursor;
+        }
+
+        public void setCursor(LogPosition cursor) {
+            this.cursor = cursor;
+        }
+
+    }
+
+    /**
+     * 描述整个canal instance对应数据对象
+     * 
+     * @author jianghang 2013-4-15 下午06:20:22
+     * @version 1.0.4
+     */
+    public static class FileMetaInstanceData {
+
+        private String                           destination;
+        private List clientDatas;
+
+        public FileMetaInstanceData(){
+
+        }
+
+        public FileMetaInstanceData(String destination, List clientDatas){
+            this.destination = destination;
+            this.clientDatas = clientDatas;
+        }
+
+        public String getDestination() {
+            return destination;
+        }
+
+        public void setDestination(String destination) {
+            this.destination = destination;
+        }
+
+        public List getClientDatas() {
+            return clientDatas;
+        }
+
+        public void setClientDatas(List clientDatas) {
+            this.clientDatas = clientDatas;
+        }
+
+    }
+
+    public void setDataDir(String dataDir) {
+        this.dataDir = new File(dataDir);
+    }
+
+    public void setDataDir(File dataDir) {
+        this.dataDir = dataDir;
+    }
+
+    public void setPeriod(long period) {
+        this.period = period;
+    }
+
+}
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java
new file mode 100644
index 00000000..6effab1a
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java
@@ -0,0 +1,226 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+
+/**
+ * 内存版实现
+ * 
+ * @author zebin.xuzb @ 2012-7-2
+ * @version 1.0.0
+ */
+public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
+
+    protected Map>              destinations;
+    protected Map batches;
+    protected Map                  cursors;
+
+    public void start() {
+        super.start();
+
+        batches = new MapMaker().makeComputingMap(new Function() {
+
+            public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
+                return MemoryClientIdentityBatch.create(clientIdentity);
+            }
+
+        });
+
+        cursors = new MapMaker().makeMap();
+
+        destinations = new MapMaker().makeComputingMap(new Function>() {
+
+            public List apply(String destination) {
+                return Lists.newArrayList();
+            }
+        });
+    }
+
+    public void stop() {
+        super.stop();
+
+        destinations.clear();
+        cursors.clear();
+        for (MemoryClientIdentityBatch batch : batches.values()) {
+            batch.clearPositionRanges();
+        }
+    }
+
+    public synchronized void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        List clientIdentitys = destinations.get(clientIdentity.getDestination());
+
+        if (clientIdentitys.contains(clientIdentity)) {
+            clientIdentitys.remove(clientIdentity);
+        }
+        
+        clientIdentitys.add(clientIdentity);
+    }
+
+    public synchronized boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        List clientIdentitys = destinations.get(clientIdentity.getDestination());
+        return clientIdentitys != null && clientIdentitys.contains(clientIdentity);
+    }
+
+    public synchronized void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        List clientIdentitys = destinations.get(clientIdentity.getDestination());
+        if (clientIdentitys != null && clientIdentitys.contains(clientIdentity)) {
+            clientIdentitys.remove(clientIdentity);
+        }
+    }
+
+    public synchronized List listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
+        return destinations.get(destination);
+    }
+
+    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        return cursors.get(clientIdentity);
+    }
+
+    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
+        cursors.put(clientIdentity, position);
+    }
+
+    public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
+        return batches.get(clientIdentity).addPositionRange(positionRange);
+    }
+
+    public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
+                                                                                                  throws CanalMetaManagerException {
+        batches.get(clientIdentity).addPositionRange(positionRange, batchId);// 添加记录到指定batchId
+    }
+
+    public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
+        return batches.get(clientIdentity).removePositionRange(batchId);
+    }
+
+    public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
+        return batches.get(clientIdentity).getPositionRange(batchId);
+    }
+
+    public PositionRange getLastestBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        return batches.get(clientIdentity).getLastestPositionRange();
+    }
+
+    public PositionRange getFirstBatch(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        return batches.get(clientIdentity).getFirstPositionRange();
+    }
+
+    public Map listAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        return batches.get(clientIdentity).listAllPositionRange();
+    }
+
+    public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        batches.get(clientIdentity).clearPositionRanges();
+    }
+
+    // ============================
+
+    public static class MemoryClientIdentityBatch {
+
+        private ClientIdentity           clientIdentity;
+        private Map batches          = new MapMaker().makeMap();
+        private AtomicLong               atomicMaxBatchId = new AtomicLong(1);
+
+        public static MemoryClientIdentityBatch create(ClientIdentity clientIdentity) {
+            return new MemoryClientIdentityBatch(clientIdentity);
+        }
+
+        public MemoryClientIdentityBatch(){
+
+        }
+
+        protected MemoryClientIdentityBatch(ClientIdentity clientIdentity){
+            this.clientIdentity = clientIdentity;
+        }
+
+        public synchronized void addPositionRange(PositionRange positionRange, Long batchId) {
+            updateMaxId(batchId);
+            batches.put(batchId, positionRange);
+        }
+
+        public synchronized Long addPositionRange(PositionRange positionRange) {
+            Long batchId = atomicMaxBatchId.getAndIncrement();
+            batches.put(batchId, positionRange);
+            return batchId;
+        }
+
+        public synchronized PositionRange removePositionRange(Long batchId) {
+            if (batches.containsKey(batchId)) {
+                Long minBatchId = Collections.min(batches.keySet());
+                if (!minBatchId.equals(batchId)) {
+                    // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
+                    throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId,
+                                                                      minBatchId));
+                }
+                return batches.remove(batchId);
+            } else {
+                return null;
+            }
+        }
+
+        public synchronized PositionRange getPositionRange(Long batchId) {
+            return batches.get(batchId);
+        }
+
+        public synchronized PositionRange getLastestPositionRange() {
+            if (batches.size() == 0) {
+                return null;
+            } else {
+                Long batchId = Collections.max(batches.keySet());
+                return batches.get(batchId);
+            }
+        }
+
+        public synchronized PositionRange getFirstPositionRange() {
+            if (batches.size() == 0) {
+                return null;
+            } else {
+                Long batchId = Collections.min(batches.keySet());
+                return batches.get(batchId);
+            }
+        }
+
+        public synchronized Map listAllPositionRange() {
+            Set batchIdSets = batches.keySet();
+            List batchIds = Lists.newArrayList(batchIdSets);
+            Collections.sort(Lists.newArrayList(batchIds));
+
+            return Maps.newHashMap(batches);
+        }
+
+        public synchronized void clearPositionRanges() {
+            batches.clear();
+        }
+
+        private synchronized void updateMaxId(Long batchId) {
+            if (atomicMaxBatchId.get() < batchId + 1) {
+                atomicMaxBatchId.set(batchId + 1);
+            }
+        }
+
+        // ============ setter & getter =========
+
+        public ClientIdentity getClientIdentity() {
+            return clientIdentity;
+        }
+
+        public void setClientIdentity(ClientIdentity clientIdentity) {
+            this.clientIdentity = clientIdentity;
+        }
+
+    }
+
+}
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java
new file mode 100644
index 00000000..84d7696d
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java
@@ -0,0 +1,185 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 组合memory + zookeeper的使用模式
+ * 
+ * @author jianghang 2012-7-11 下午03:58:00
+ * @version 1.0.0
+ */
+
+public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
+
+    private ExecutorService      executor;
+    private ZooKeeperMetaManager zooKeeperMetaManager;
+    @SuppressWarnings("serial")
+    private final Position       nullCursor = new Position() {
+                                            };
+
+    public void start() {
+        super.start();
+        Assert.notNull(zooKeeperMetaManager);
+        if (!zooKeeperMetaManager.isStart()) {
+            zooKeeperMetaManager.start();
+        }
+
+        executor = Executors.newFixedThreadPool(1);
+        destinations = new MapMaker().makeComputingMap(new Function>() {
+
+            public List apply(String destination) {
+                return zooKeeperMetaManager.listAllSubscribeInfo(destination);
+            }
+        });
+
+        cursors = new MapMaker().makeComputingMap(new Function() {
+
+            public Position apply(ClientIdentity clientIdentity) {
+                Position position = zooKeeperMetaManager.getCursor(clientIdentity);
+                if (position == null) {
+                    return nullCursor; // 返回一个空对象标识，避免出现异常
+                } else {
+                    return position;
+                }
+            }
+        });
+
+        batches = new MapMaker().makeComputingMap(new Function() {
+
+            public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
+                // 读取一下zookeeper信息，初始化一次
+                MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
+                Map positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
+                for (Map.Entry entry : positionRanges.entrySet()) {
+                    batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
+                }
+                return batches;
+            }
+        });
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zooKeeperMetaManager.isStart()) {
+            zooKeeperMetaManager.stop();
+        }
+
+        executor.shutdownNow();
+        destinations.clear();
+        batches.clear();
+    }
+
+    public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.subscribe(clientIdentity);
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.subscribe(clientIdentity);
+            }
+        });
+    }
+
+    public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.unsubscribe(clientIdentity);
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.unsubscribe(clientIdentity);
+            }
+        });
+    }
+
+    public void updateCursor(final ClientIdentity clientIdentity, final Position position)
+                                                                                          throws CanalMetaManagerException {
+        super.updateCursor(clientIdentity, position);
+
+        // 异步刷新
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.updateCursor(clientIdentity, position);
+            }
+        });
+    }
+
+    @Override
+    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        Position position = super.getCursor(clientIdentity);
+        if (position == nullCursor) {
+            return null;
+        } else {
+            return position;
+        }
+    }
+
+    public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
+                                                                                                throws CanalMetaManagerException {
+        final Long batchId = super.addBatch(clientIdentity, positionRange);
+        // 异步刷新
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
+            }
+        });
+        return batchId;
+    }
+
+    public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
+                                                                                                                    throws CanalMetaManagerException {
+        super.addBatch(clientIdentity, positionRange, batchId);
+        // 异步刷新
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
+            }
+        });
+    }
+
+    public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
+                                                                                             throws CanalMetaManagerException {
+        PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
+        // 异步刷新
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
+            }
+        });
+
+        return positionRange;
+    }
+
+    public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.clearAllBatchs(clientIdentity);
+
+        // 异步刷新
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.clearAllBatchs(clientIdentity);
+            }
+        });
+    }
+
+    // =============== setter / getter ================
+    public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
+        this.zooKeeperMetaManager = zooKeeperMetaManager;
+    }
+}
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java
new file mode 100644
index 00000000..c4c978d7
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java
@@ -0,0 +1,168 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于定时刷新的策略的mixed实现
+ * 
+ * 
+ * 几个优化：
+ * 1. 去除batch数据刷新到zk中，切换时batch数据可忽略，重新从头开始获取
+ * 2. cursor的更新，启用定时刷新，合并多次请求。如果最近没有变化则不更新
+ * 

+ * 
+ * @author jianghang 2012-9-11 下午02:41:15
+ * @version 1.0.0
+ */
+public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
+
+    private static final Logger      logger     = LoggerFactory.getLogger(PeriodMixedMetaManager.class);
+    private ScheduledExecutorService executor;
+    private ZooKeeperMetaManager     zooKeeperMetaManager;
+    @SuppressWarnings("serial")
+    private final Position           nullCursor = new Position() {
+                                                };
+    private long                     period     = 1000;                                                 // 单位ms
+    private Set      updateCursorTasks;
+
+    public void start() {
+        super.start();
+        Assert.notNull(zooKeeperMetaManager);
+        if (!zooKeeperMetaManager.isStart()) {
+            zooKeeperMetaManager.start();
+        }
+
+        executor = Executors.newScheduledThreadPool(1);
+        destinations = new MapMaker().makeComputingMap(new Function>() {
+
+            public List apply(String destination) {
+                return zooKeeperMetaManager.listAllSubscribeInfo(destination);
+            }
+        });
+
+        cursors = new MapMaker().makeComputingMap(new Function() {
+
+            public Position apply(ClientIdentity clientIdentity) {
+                Position position = zooKeeperMetaManager.getCursor(clientIdentity);
+                if (position == null) {
+                    return nullCursor; // 返回一个空对象标识，避免出现异常
+                } else {
+                    return position;
+                }
+            }
+        });
+
+        batches = new MapMaker().makeComputingMap(new Function() {
+
+            public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
+                // 读取一下zookeeper信息，初始化一次
+                MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
+                Map positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
+                for (Map.Entry entry : positionRanges.entrySet()) {
+                    batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
+                }
+                return batches;
+            }
+        });
+
+        updateCursorTasks = Collections.synchronizedSet(new HashSet());
+
+        // 启动定时工作任务
+        executor.scheduleAtFixedRate(new Runnable() {
+
+            public void run() {
+                List tasks = new ArrayList(updateCursorTasks);
+                for (ClientIdentity clientIdentity : tasks) {
+                    try {
+                        // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
+                        zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
+                        updateCursorTasks.remove(clientIdentity);
+                    } catch (Throwable e) {
+                        // ignore
+                        logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
+                    }
+                }
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zooKeeperMetaManager.isStart()) {
+            zooKeeperMetaManager.stop();
+        }
+
+        executor.shutdownNow();
+        destinations.clear();
+        batches.clear();
+    }
+
+    public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.subscribe(clientIdentity);
+
+        // 订阅信息频率发生比较低，不需要做定时merge处理
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.subscribe(clientIdentity);
+            }
+        });
+    }
+
+    public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        super.unsubscribe(clientIdentity);
+
+        // 订阅信息频率发生比较低，不需要做定时merge处理
+        executor.submit(new Runnable() {
+
+            public void run() {
+                zooKeeperMetaManager.unsubscribe(clientIdentity);
+            }
+        });
+    }
+
+    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
+        updateCursorTasks.add(clientIdentity);// 添加到任务队列中进行触发
+        super.updateCursor(clientIdentity, position);
+    }
+
+    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        Position position = super.getCursor(clientIdentity);
+        if (position == nullCursor) {
+            return null;
+        } else {
+            return position;
+        }
+    }
+
+    // =============== setter / getter ================
+
+    public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
+        this.zooKeeperMetaManager = zooKeeperMetaManager;
+    }
+
+    public void setPeriod(long period) {
+        this.period = period;
+    }
+
+}
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java b/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
new file mode 100644
index 00000000..38b459c9
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManager.java
@@ -0,0 +1,329 @@
+package com.alibaba.otter.canal.meta;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.commons.lang.StringUtils;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * zk 版本的 canal manager， 存储结构：
+ * 
+ * 
+ * /otter
+ *    canal
+ *      destinations
+ *        dest1 
+ *          client1
+ *            filter
+ *            batch_mark
+ *              1
+ *              2
+ *              3
+ * 

+ * 
+ * @author zebin.xuzb @ 2012-6-21
+ * @author jianghang
+ * @version 1.0.0
+ */
+public class ZooKeeperMetaManager extends AbstractCanalLifeCycle implements CanalMetaManager {
+
+    private static final String ENCODE = "UTF-8";
+    private ZkClientx           zkClientx;
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(zkClientx);
+    }
+
+    public void stop() {
+        super.stop();
+    }
+
+    public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+
+        try {
+            zkClientx.createPersistent(path, true);
+        } catch (ZkNodeExistsException e) {
+            // ignore
+        }
+        if (clientIdentity.hasFilter()) {
+            String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(),
+                clientIdentity.getClientId());
+
+            byte[] bytes = null;
+            try {
+                bytes = clientIdentity.getFilter().getBytes(ENCODE);
+            } catch (UnsupportedEncodingException e) {
+                throw new CanalMetaManagerException(e);
+            }
+
+            try {
+                zkClientx.createPersistent(filterPath, bytes);
+            } catch (ZkNodeExistsException e) {
+                // ignore
+                zkClientx.writeData(filterPath, bytes);
+            }
+        }
+    }
+
+    public boolean hasSubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        return zkClientx.exists(path);
+    }
+
+    public void unsubscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        zkClientx.deleteRecursive(path); // 递归删除所有信息
+    }
+
+    public List listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        List childs = null;
+        try {
+            childs = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(childs)) {
+            return new ArrayList();
+        }
+        List clientIds = new ArrayList();
+        for (String child : childs) {
+            if (StringUtils.isNumeric(child)) {
+                clientIds.add(ZookeeperPathUtils.getClientId(child));
+            }
+        }
+
+        Collections.sort(clientIds); // 进行一个排序
+        List clientIdentities = Lists.newArrayList();
+        for (Short clientId : clientIds) {
+            path = ZookeeperPathUtils.getFilterPath(destination, clientId);
+            byte[] bytes = zkClientx.readData(path, true);
+            String filter = null;
+            if (bytes != null) {
+                try {
+                    filter = new String(bytes, ENCODE);
+                } catch (UnsupportedEncodingException e) {
+                    throw new CanalMetaManagerException(e);
+                }
+            }
+            clientIdentities.add(new ClientIdentity(destination, clientId, filter));
+        }
+
+        return clientIdentities;
+    }
+
+    public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+
+        byte[] data = zkClientx.readData(path, true);
+        if (data == null || data.length == 0) {
+            return null;
+        }
+
+        return JsonUtils.unmarshalFromByte(data, Position.class);
+    }
+
+    public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
+        try {
+            zkClientx.writeData(path, data);
+        } catch (ZkNoNodeException e) {
+            zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
+        }
+    }
+
+    public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
+        String batchPath = zkClientx.createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR,
+            data,
+            true);
+        String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
+        return ZookeeperPathUtils.getBatchMarkId(batchIdString);
+    }
+
+    public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange, Long batchId)
+                                                                                                  throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId(),
+            batchId);
+        byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
+        zkClientx.createPersistent(path, data, true);
+    }
+
+    public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
+        String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId());
+        List nodes = zkClientx.getChildren(batchsPath);
+        if (CollectionUtils.isEmpty(nodes)) {
+            // 没有batch记录
+            return null;
+        }
+
+        // 找到最小的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+        Long minBatchId = Collections.min(batchIds);
+        if (!minBatchId.equals(batchId)) {
+            // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
+            throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
+        }
+
+        if (!batchIds.contains(batchId)) {
+            // 不存在对应的batchId
+            return null;
+        }
+        PositionRange positionRange = getBatch(clientIdentity, batchId);
+        if (positionRange != null) {
+            String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
+                clientIdentity.getClientId(),
+                batchId);
+            zkClientx.delete(path);
+        }
+
+        return positionRange;
+    }
+
+    public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(),
+            clientIdentity.getClientId(),
+            batchId);
+        byte[] data = zkClientx.readData(path, true);
+        if (data == null) {
+            return null;
+        }
+
+        PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
+        return positionRange;
+    }
+
+    public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        List batchChilds = zkClientx.getChildren(path);
+
+        for (String batchChild : batchChilds) {
+            String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
+            zkClientx.delete(batchPath);
+        }
+    }
+
+    public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        List nodes = null;
+        try {
+            nodes = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(nodes)) {
+            return null;
+        }
+        // 找到最大的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+        Long maxBatchId = Collections.max(batchIds);
+        PositionRange result = getBatch(clientIdentity, maxBatchId);
+        if (result == null) { // 出现为null，说明zk节点有变化，重新获取
+            return getLastestBatch(clientIdentity);
+        } else {
+            return result;
+        }
+    }
+
+    public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        List nodes = null;
+        try {
+            nodes = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(nodes)) {
+            return null;
+        }
+        // 找到最小的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+        Long minBatchId = Collections.min(batchIds);
+        PositionRange result = getBatch(clientIdentity, minBatchId);
+        if (result == null) { // 出现为null，说明zk节点有变化，重新获取
+            return getFirstBatch(clientIdentity);
+        } else {
+            return result;
+        }
+    }
+
+    public Map listAllBatchs(ClientIdentity clientIdentity) {
+        String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
+        List nodes = null;
+        try {
+            nodes = zkClientx.getChildren(path);
+        } catch (ZkNoNodeException e) {
+            // ignore
+        }
+
+        if (CollectionUtils.isEmpty(nodes)) {
+            return Maps.newHashMap();
+        }
+        // 找到最大的Id
+        ArrayList batchIds = new ArrayList(nodes.size());
+        for (String batchIdString : nodes) {
+            batchIds.add(Long.valueOf(batchIdString));
+        }
+
+        Collections.sort(batchIds); // 从小到大排序
+        Map positionRanges = Maps.newLinkedHashMap();
+        for (Long batchId : batchIds) {
+            PositionRange result = getBatch(clientIdentity, batchId);
+            if (result == null) {// 出现为null，说明zk节点有变化，重新获取
+                return listAllBatchs(clientIdentity);
+            } else {
+                positionRanges.put(batchId, result);
+            }
+        }
+
+        return positionRanges;
+    }
+
+    // =========== setter ==========
+
+    public void setZkClientx(ZkClientx zkClientx) {
+        this.zkClientx = zkClientx;
+    }
+
+}
diff --git a/meta/src/main/java/com/alibaba/otter/canal/meta/exception/CanalMetaManagerException.java b/meta/src/main/java/com/alibaba/otter/canal/meta/exception/CanalMetaManagerException.java
new file mode 100644
index 00000000..2f162392
--- /dev/null
+++ b/meta/src/main/java/com/alibaba/otter/canal/meta/exception/CanalMetaManagerException.java
@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.meta.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * @author zebin.xuzb @ 2012-6-21
+ * @version 1.0.0
+ */
+public class CanalMetaManagerException extends CanalException {
+
+    private static final long serialVersionUID = -654893533794556357L;
+
+    public CanalMetaManagerException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalMetaManagerException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalMetaManagerException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalMetaManagerException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalMetaManagerException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java
new file mode 100644
index 00000000..2e83921c
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/AbstractMetaManagerTest.java
@@ -0,0 +1,117 @@
+package com.alibaba.otter.canal.meta;
+
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class AbstractMetaManagerTest extends AbstractZkTest {
+
+    private static final String MYSQL_ADDRESS  = "127.0.0.1";
+    protected ClientIdentity    clientIdentity = new ClientIdentity(destination, (short) 1); ;
+
+    public void doSubscribeTest(CanalMetaManager metaManager) {
+        ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
+        metaManager.subscribe(client1);
+        metaManager.subscribe(client1); // 重复调用
+        ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
+        metaManager.subscribe(client2);
+
+        List clients = metaManager.listAllSubscribeInfo(destination);
+        Assert.assertEquals(Arrays.asList(client1, client2), clients);
+
+        metaManager.unsubscribe(client2);
+        ClientIdentity client3 = new ClientIdentity(destination, (short) 3);
+        metaManager.subscribe(client3);
+
+        clients = metaManager.listAllSubscribeInfo(destination);
+        Assert.assertEquals(Arrays.asList(client1, client3), clients);
+
+    }
+
+    public void doBatchTest(CanalMetaManager metaManager) {
+        metaManager.subscribe(clientIdentity);
+
+        PositionRange first = metaManager.getFirstBatch(clientIdentity);
+        PositionRange lastest = metaManager.getLastestBatch(clientIdentity);
+
+        Assert.assertNull(first);
+        Assert.assertNull(lastest);
+
+        PositionRange range1 = buildRange(1);
+        Long batchId1 = metaManager.addBatch(clientIdentity, range1);
+
+        PositionRange range2 = buildRange(2);
+        Long batchId2 = metaManager.addBatch(clientIdentity, range2);
+        Assert.assertEquals((batchId1.longValue() + 1), batchId2.longValue());
+
+        // 验证get
+        PositionRange getRange1 = metaManager.getBatch(clientIdentity, batchId1);
+        Assert.assertEquals(range1, getRange1);
+        PositionRange getRange2 = metaManager.getBatch(clientIdentity, batchId2);
+        Assert.assertEquals(range2, getRange2);
+
+        PositionRange range3 = buildRange(3);
+        Long batchId3 = batchId2 + 1;
+        metaManager.addBatch(clientIdentity, range3, batchId3);
+
+        PositionRange range4 = buildRange(4);
+        Long batchId4 = metaManager.addBatch(clientIdentity, range4);
+        Assert.assertEquals((batchId3.longValue() + 1), batchId4.longValue());
+
+        // 验证remove
+        metaManager.removeBatch(clientIdentity, batchId1);
+        range1 = metaManager.getBatch(clientIdentity, batchId1);
+        Assert.assertNull(range1);
+
+        // 验证first / lastest
+        first = metaManager.getFirstBatch(clientIdentity);
+        lastest = metaManager.getLastestBatch(clientIdentity);
+
+        Assert.assertEquals(range2, first);
+        Assert.assertEquals(range4, lastest);
+
+        Map ranges = metaManager.listAllBatchs(clientIdentity);
+        Assert.assertEquals(3, ranges.size());
+    }
+
+    public Position doCursorTest(CanalMetaManager metaManager) {
+        metaManager.subscribe(clientIdentity);
+
+        Position position1 = metaManager.getCursor(clientIdentity);
+        Assert.assertNull(position1);
+
+        PositionRange range = buildRange(1);
+
+        metaManager.updateCursor(clientIdentity, range.getStart());
+        Position position2 = metaManager.getCursor(clientIdentity);
+        Assert.assertEquals(range.getStart(), position2);
+
+        metaManager.updateCursor(clientIdentity, range.getEnd());
+        Position position3 = metaManager.getCursor(clientIdentity);
+        Assert.assertEquals(range.getEnd(), position3);
+
+        return position3;
+    }
+
+    private PositionRange buildRange(int number) {
+        LogPosition start = new LogPosition();
+        start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
+
+        LogPosition end = new LogPosition();
+        end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
+        return new PositionRange(start, end);
+    }
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/AbstractZkTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/AbstractZkTest.java
new file mode 100644
index 00000000..dd59b163
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/AbstractZkTest.java
@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.meta;
+
+import org.junit.Assert;
+
+public class AbstractZkTest {
+
+    protected String destination = "ljhtest1";
+    protected String cluster1    = "127.0.0.1:2188";
+    protected String cluster2    = "127.0.0.1:2188,127.0.0.1:2188";
+
+    public void sleep(long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java
new file mode 100644
index 00000000..eb97cf94
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/FileMixedMetaManagerTest.java
@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.meta;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class FileMixedMetaManagerTest extends AbstractMetaManagerTest {
+
+    private static final String tmp     = System.getProperty("java.io.tmpdir", "/tmp");
+    private static final File   dataDir = new File(tmp, "canal");
+
+    @Before
+    public void setUp() {
+        try {
+            FileUtils.deleteDirectory(dataDir);
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+
+    @Test
+    public void testSubscribeAll() {
+        FileMixedMetaManager metaManager = new FileMixedMetaManager();
+        metaManager.setDataDir(dataDir);
+        metaManager.setPeriod(100);
+
+        metaManager.start();
+        doSubscribeTest(metaManager);
+
+        sleep(2000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
+        metaManager2.setDataDir(dataDir);
+        metaManager2.setPeriod(100);
+        metaManager2.start();
+
+        List clients = metaManager2.listAllSubscribeInfo(destination);
+        Assert.assertEquals(2, clients.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testBatchAll() {
+        FileMixedMetaManager metaManager = new FileMixedMetaManager();
+        metaManager.setDataDir(dataDir);
+        metaManager.setPeriod(100);
+
+        metaManager.start();
+        doBatchTest(metaManager);
+
+        metaManager.clearAllBatchs(clientIdentity);
+        Map ranges = metaManager.listAllBatchs(clientIdentity);
+        Assert.assertEquals(0, ranges.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testCursorAll() {
+        FileMixedMetaManager metaManager = new FileMixedMetaManager();
+        metaManager.setDataDir(dataDir);
+        metaManager.setPeriod(100);
+        metaManager.start();
+
+        Position lastPosition = doCursorTest(metaManager);
+
+        sleep(1000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
+        metaManager2.setDataDir(dataDir);
+        metaManager2.setPeriod(100);
+        metaManager2.start();
+
+        Position position = metaManager2.getCursor(clientIdentity);
+        Assert.assertEquals(position, lastPosition);
+        metaManager.stop();
+    }
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java
new file mode 100644
index 00000000..fa8d7d7f
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/MemoryMetaManagerTest.java
@@ -0,0 +1,40 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class MemoryMetaManagerTest extends AbstractMetaManagerTest {
+
+    @Test
+    public void testSubscribeAll() {
+        MemoryMetaManager metaManager = new MemoryMetaManager();
+        metaManager.start();
+        doSubscribeTest(metaManager);
+        metaManager.stop();
+    }
+
+    @Test
+    public void testBatchAll() {
+        MemoryMetaManager metaManager = new MemoryMetaManager();
+        metaManager.start();
+        doBatchTest(metaManager);
+
+        metaManager.clearAllBatchs(clientIdentity);
+        Map ranges = metaManager.listAllBatchs(clientIdentity);
+        Assert.assertEquals(0, ranges.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testCursorAll() {
+        MemoryMetaManager metaManager = new MemoryMetaManager();
+        metaManager.start();
+        doCursorTest(metaManager);
+        metaManager.stop();
+    }
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/MixedMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/MixedMetaManagerTest.java
new file mode 100644
index 00000000..947a31c8
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/MixedMetaManagerTest.java
@@ -0,0 +1,104 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class MixedMetaManagerTest extends AbstractMetaManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testSubscribeAll() {
+        MixedMetaManager metaManager = new MixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+        doSubscribeTest(metaManager);
+
+        sleep(1000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        MixedMetaManager metaManager2 = new MixedMetaManager();
+        metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager2.start();
+
+        List clients = metaManager2.listAllSubscribeInfo(destination);
+        Assert.assertEquals(2, clients.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testBatchAll() {
+        MixedMetaManager metaManager = new MixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+        doBatchTest(metaManager);
+
+        sleep(1000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        MixedMetaManager metaManager2 = new MixedMetaManager();
+        metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager2.start();
+
+        Map ranges = metaManager2.listAllBatchs(clientIdentity);
+        Assert.assertEquals(3, ranges.size());
+
+        metaManager.clearAllBatchs(clientIdentity);
+        ranges = metaManager.listAllBatchs(clientIdentity);
+        Assert.assertEquals(0, ranges.size());
+        metaManager.stop();
+        metaManager2.stop();
+    }
+
+    @Test
+    public void testCursorAll() {
+        MixedMetaManager metaManager = new MixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+        Position lastPosition = doCursorTest(metaManager);
+
+        sleep(1000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        MixedMetaManager metaManager2 = new MixedMetaManager();
+        metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager2.start();
+
+        Position position = metaManager2.getCursor(clientIdentity);
+        Assert.assertEquals(position, lastPosition);
+        metaManager.stop();
+    }
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManagerTest.java
new file mode 100644
index 00000000..b63f4787
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManagerTest.java
@@ -0,0 +1,94 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class PeriodMixedMetaManagerTest extends AbstractMetaManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testSubscribeAll() {
+        PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+        doSubscribeTest(metaManager);
+
+        sleep(1000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        PeriodMixedMetaManager metaManager2 = new PeriodMixedMetaManager();
+        metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager2.start();
+
+        List clients = metaManager2.listAllSubscribeInfo(destination);
+        Assert.assertEquals(2, clients.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testBatchAll() {
+        PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+        doBatchTest(metaManager);
+
+        metaManager.clearAllBatchs(clientIdentity);
+        Map ranges = metaManager.listAllBatchs(clientIdentity);
+        Assert.assertEquals(0, ranges.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testCursorAll() {
+        PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+        Position lastPosition = doCursorTest(metaManager);
+
+        sleep(1000L);
+        // 重新构建一次，能获得上一次zk上的记录
+        PeriodMixedMetaManager metaManager2 = new PeriodMixedMetaManager();
+        metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager2.start();
+
+        Position position = metaManager2.getCursor(clientIdentity);
+        Assert.assertEquals(position, lastPosition);
+        metaManager.stop();
+    }
+}
diff --git a/meta/src/test/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManagerTest.java b/meta/src/test/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManagerTest.java
new file mode 100644
index 00000000..85cce8e7
--- /dev/null
+++ b/meta/src/test/java/com/alibaba/otter/canal/meta/ZooKeeperMetaManagerTest.java
@@ -0,0 +1,62 @@
+package com.alibaba.otter.canal.meta;
+
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class ZooKeeperMetaManagerTest extends AbstractMetaManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testSubscribeAll() {
+        ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
+        metaManager.setZkClientx(zkclientx);
+        metaManager.start();
+
+        doSubscribeTest(metaManager);
+        metaManager.stop();
+    }
+
+    @Test
+    public void testBatchAll() {
+        ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
+        metaManager.setZkClientx(zkclientx);
+        metaManager.start();
+        doBatchTest(metaManager);
+
+        metaManager.clearAllBatchs(clientIdentity);
+        Map ranges = metaManager.listAllBatchs(clientIdentity);
+        Assert.assertEquals(0, ranges.size());
+        metaManager.stop();
+    }
+
+    @Test
+    public void testCursorhAll() {
+        ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
+        metaManager.setZkClientx(zkclientx);
+        metaManager.start();
+        doCursorTest(metaManager);
+        metaManager.stop();
+    }
+}
diff --git a/parse/pom.xml b/parse/pom.xml
new file mode 100644
index 00000000..c495d693
--- /dev/null
+++ b/parse/pom.xml
@@ -0,0 +1,56 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.parse
+	jar
+	canal parse module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.protocol
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.meta
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.sink
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.parse.dbsync
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.filter
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.parse.driver
+			${project.version}
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/CanalEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/CanalEventParser.java
new file mode 100644
index 00000000..e9167e84
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/CanalEventParser.java
@@ -0,0 +1,13 @@
+package com.alibaba.otter.canal.parse;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+
+/**
+ * 数据复制控制器
+ * 
+ * @author jianghang 2012-6-21 下午04:03:25
+ * @version 1.0.0
+ */
+public interface CanalEventParser extends CanalLifeCycle {
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/CanalHASwitchable.java b/parse/src/main/java/com/alibaba/otter/canal/parse/CanalHASwitchable.java
new file mode 100644
index 00000000..38113d3d
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/CanalHASwitchable.java
@@ -0,0 +1,16 @@
+package com.alibaba.otter.canal.parse;
+
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+
+/**
+ * 支持可切换的数据复制控制器
+ * 
+ * @author jianghang 2012-6-26 下午05:41:43
+ * @version 1.0.0
+ */
+public interface CanalHASwitchable {
+
+    public void doSwitch();
+
+    public void doSwitch(AuthenticationInfo newAuthenticationInfo);
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalHAException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalHAException.java
new file mode 100644
index 00000000..e5735461
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalHAException.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalHAException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalHAException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalHAException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalHAException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalHAException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalHAException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalParseException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalParseException.java
new file mode 100644
index 00000000..e897835f
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/CanalParseException.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalParseException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalParseException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalParseException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalParseException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalParseException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalParseException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/exception/TableIdNotFoundException.java b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/TableIdNotFoundException.java
new file mode 100644
index 00000000..aff34d7f
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/exception/TableIdNotFoundException.java
@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.parse.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+public class TableIdNotFoundException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public TableIdNotFoundException(String errorCode){
+        super(errorCode);
+    }
+
+    public TableIdNotFoundException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public TableIdNotFoundException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public TableIdNotFoundException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public TableIdNotFoundException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/ha/CanalHAController.java b/parse/src/main/java/com/alibaba/otter/canal/parse/ha/CanalHAController.java
new file mode 100644
index 00000000..0724d89d
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/ha/CanalHAController.java
@@ -0,0 +1,17 @@
+package com.alibaba.otter.canal.parse.ha;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalHAException;
+
+/**
+ * HA 控制器实现
+ * 
+ * @author jianghang 2012-6-26 下午05:21:07
+ * @version 1.0.0
+ */
+public interface CanalHAController extends CanalLifeCycle {
+
+    public void start() throws CanalHAException;
+
+    public void stop() throws CanalHAException;
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/ha/HeartBeatHAController.java b/parse/src/main/java/com/alibaba/otter/canal/parse/ha/HeartBeatHAController.java
new file mode 100644
index 00000000..d7fee185
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/ha/HeartBeatHAController.java
@@ -0,0 +1,63 @@
+package com.alibaba.otter.canal.parse.ha;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.CanalHASwitchable;
+import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback;
+
+/**
+ * 基于HeartBeat信息的HA控制 , 注意：非线程安全，需要做做多例化
+ * 
+ * @author jianghang 2012-7-6 下午02:33:30
+ * @version 1.0.0
+ */
+public class HeartBeatHAController extends AbstractCanalLifeCycle implements CanalHAController, HeartBeatCallback {
+
+    private static final Logger logger              = LoggerFactory.getLogger(HeartBeatHAController.class);
+    // default 3 times
+    private int                 detectingRetryTimes = 3;
+    private int                 failedTimes         = 0;
+    private boolean             switchEnable        = false;
+    private CanalHASwitchable   eventParser;
+
+    public HeartBeatHAController(){
+
+    }
+
+    public void onSuccess(long costTime) {
+        failedTimes = 0;
+    }
+
+    public void onFailed(Throwable e) {
+        failedTimes++;
+        // 检查一下是否超过失败次数
+        synchronized (this) {
+            if (failedTimes > detectingRetryTimes) {
+                if (switchEnable) {
+                    eventParser.doSwitch();// 通知执行一次切换
+                    failedTimes = 0;
+                } else {
+                    logger.warn("HeartBeat failed Times:{} , should auto switch ?", failedTimes);
+                }
+            }
+        }
+    }
+
+    // ============================= setter / getter
+    // ============================
+
+    public void setCanalHASwitchable(CanalHASwitchable canalHASwitchable) {
+        this.eventParser = canalHASwitchable;
+    }
+
+    public void setDetectingRetryTimes(int detectingRetryTimes) {
+        this.detectingRetryTimes = detectingRetryTimes;
+    }
+
+    public void setSwitchEnable(boolean switchEnable) {
+        this.switchEnable = switchEnable;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractBinlogParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractBinlogParser.java
new file mode 100644
index 00000000..6024114c
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractBinlogParser.java
@@ -0,0 +1,25 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+
+public abstract class AbstractBinlogParser extends AbstractCanalLifeCycle implements BinlogParser {
+
+    public void reset() {
+    }
+
+    public Entry parse(T event, TableMeta tableMeta) throws CanalParseException {
+        return null;
+    }
+
+    public Entry parse(T event) throws CanalParseException {
+        return null;
+    }
+
+    public void stop() {
+        reset();
+        super.stop();
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
new file mode 100644
index 00000000..6be3de39
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/AbstractEventParser.java
@@ -0,0 +1,519 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang.math.RandomUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.alarm.CanalAlarmHandler;
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
+import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+/**
+ * 抽象的EventParser, 最大化共用mysql/oracle版本的实现
+ * 
+ * @author jianghang 2013-1-20 下午08:10:25
+ * @version 1.0.0
+ */
+public abstract class AbstractEventParser extends AbstractCanalLifeCycle implements CanalEventParser {
+
+    protected final Logger                           logger                     = LoggerFactory.getLogger(this.getClass());
+
+    protected CanalLogPositionManager                logPositionManager         = null;
+    protected CanalEventSink> eventSink                  = null;
+    protected CanalEventFilter                       eventFilter                = null;
+    protected CanalEventFilter                       eventBlackFilter           = null;
+
+    private CanalAlarmHandler                        alarmHandler               = null;
+
+    // 统计参数
+    protected AtomicBoolean                          profilingEnabled           = new AtomicBoolean(false);                // profile开关参数
+    protected AtomicLong                             receivedEventCount         = new AtomicLong();
+    protected AtomicLong                             parsedEventCount           = new AtomicLong();
+    protected AtomicLong                             consumedEventCount         = new AtomicLong();
+    protected long                                   parsingInterval            = -1;
+    protected long                                   processingInterval         = -1;
+
+    // 认证信息
+    protected volatile AuthenticationInfo            runningInfo;
+    protected String                                 destination;
+
+    // binLogParser
+    protected BinlogParser                           binlogParser               = null;
+
+    protected Thread                                 parseThread                = null;
+
+    protected Thread.UncaughtExceptionHandler        handler                    = new Thread.UncaughtExceptionHandler() {
+
+                                                                                    public void uncaughtException(Thread t,
+                                                                                                                  Throwable e) {
+                                                                                        logger.error("parse events has an error",
+                                                                                            e);
+                                                                                    }
+                                                                                };
+
+    protected EventTransactionBuffer                 transactionBuffer;
+    protected int                                    transactionSize            = 1024;
+    protected AtomicBoolean                          needTransactionPosition    = new AtomicBoolean(false);
+    protected long                                   lastEntryTime              = 0L;
+    protected volatile boolean                       detectingEnable            = true;                                    // 是否开启心跳检查
+    protected Integer                                detectingIntervalInSeconds = 3;                                       // 检测频率
+    protected volatile Timer                         timer;
+    protected TimerTask                              heartBeatTimerTask;
+    protected Throwable                              exception                  = null;
+
+    protected abstract BinlogParser buildParser();
+
+    protected abstract ErosaConnection buildErosaConnection();
+
+    protected abstract EntryPosition findStartPosition(ErosaConnection connection) throws IOException;
+
+    protected void preDump(ErosaConnection connection) {
+    }
+
+    protected void afterDump(ErosaConnection connection) {
+    }
+
+    public void sendAlarm(String destination, String msg) {
+        if (this.alarmHandler != null) {
+            this.alarmHandler.sendAlarm(destination, msg);
+        }
+    }
+
+    public AbstractEventParser(){
+        // 初始化一下
+        transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {
+
+            public void flush(List transaction) throws InterruptedException {
+                boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
+                if (!running) {
+                    return;
+                }
+
+                if (!successed) {
+                    throw new CanalParseException("consume failed!");
+                }
+
+                LogPosition position = buildLastTranasctionPosition(transaction);
+                if (position != null) { // 可能position为空
+                    logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
+                }
+            }
+        });
+    }
+
+    public void start() {
+        super.start();
+        MDC.put("destination", destination);
+        // 配置transaction buffer
+        // 初始化缓冲队列
+        transactionBuffer.setBufferSize(transactionSize);// 设置buffer大小
+        transactionBuffer.start();
+        // 构造bin log parser
+        binlogParser = buildParser();// 初始化一下BinLogParser
+        binlogParser.start();
+        // 启动工作线程
+        parseThread = new Thread(new Runnable() {
+
+            public void run() {
+                MDC.put("destination", String.valueOf(destination));
+                ErosaConnection erosaConnection = null;
+                while (running) {
+                    try {
+
+                        // 开始执行replication
+                        // 1. 构造Erosa连接
+                        erosaConnection = buildErosaConnection();
+
+                        // 2. 启动一个心跳线程
+                        startHeartBeat(erosaConnection);
+
+                        // 3. 执行dump前的准备工作
+                        preDump(erosaConnection);
+
+                        erosaConnection.connect();// 链接
+                        // 4. 获取最后的位置信息
+                        final EntryPosition startPosition = findStartPosition(erosaConnection);
+                        if (startPosition == null) {
+                            throw new CanalParseException("can't find start position for " + destination);
+                        }
+                        logger.info("find start position : {}", startPosition.toString());
+                        // 重新链接，因为在找position过程中可能有状态，需要断开后重建
+                        erosaConnection.reconnect();
+
+                        final SinkFunction sinkHandler = new SinkFunction() {
+
+                            private LogPosition lastPosition;
+
+                            public boolean sink(EVENT event) {
+                                try {
+                                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+
+                                    if (!running) {
+                                        return false;
+                                    }
+
+                                    if (entry != null) {
+                                        exception = null; // 有正常数据流过，清空exception
+                                        transactionBuffer.add(entry);
+                                        // 记录一下对应的positions
+                                        this.lastPosition = buildLastPosition(entry);
+                                        // 记录一下最后一次有数据的时间
+                                        lastEntryTime = System.currentTimeMillis();
+                                    }
+                                    return running;
+                                } catch (TableIdNotFoundException e) {
+                                    throw e;
+                                } catch (Exception e) {
+                                    // 记录一下，出错的位点信息
+                                    processError(e,
+                                        this.lastPosition,
+                                        startPosition.getJournalName(),
+                                        startPosition.getPosition());
+                                    throw new CanalParseException(e); // 继续抛出异常，让上层统一感知
+                                }
+                            }
+
+                        };
+
+                        // 4. 开始dump数据
+                        if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
+                            erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
+                        } else {
+                            erosaConnection.dump(startPosition.getJournalName(),
+                                startPosition.getPosition(),
+                                sinkHandler);
+                        }
+
+                    } catch (TableIdNotFoundException e) {
+                        exception = e;
+                        // 特殊处理TableIdNotFound异常,出现这样的异常，一种可能就是起始的position是一个事务当中，导致tablemap
+                        // Event时间没解析过
+                        needTransactionPosition.compareAndSet(false, true);
+                        logger.error(String.format("dump address %s has an error, retrying. caused by ",
+                            runningInfo.getAddress().toString()), e);
+                    } catch (Throwable e) {
+                        exception = e;
+                        if (!running) {
+                            if (!(e instanceof java.nio.channels.ClosedByInterruptException || e.getCause() instanceof java.nio.channels.ClosedByInterruptException)) {
+                                throw new CanalParseException(String.format("dump address %s has an error, retrying. ",
+                                    runningInfo.getAddress().toString()), e);
+                            }
+                        } else {
+                            logger.error(String.format("dump address %s has an error, retrying. caused by ",
+                                runningInfo.getAddress().toString()), e);
+                            sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
+                        }
+                    } finally {
+                        // 关闭一下链接
+                        afterDump(erosaConnection);
+                        try {
+                            if (erosaConnection != null) {
+                                erosaConnection.disconnect();
+                            }
+                        } catch (IOException e1) {
+                            if (!running) {
+                                throw new CanalParseException(String.format("disconnect address %s has an error, retrying. ",
+                                    runningInfo.getAddress().toString()),
+                                    e1);
+                            } else {
+                                logger.error("disconnect address {} has an error, retrying., caused by ",
+                                    runningInfo.getAddress().toString(),
+                                    e1);
+                            }
+                        }
+                    }
+                    // 出异常了，退出sink消费，释放一下状态
+                    eventSink.interrupt();
+                    transactionBuffer.reset();// 重置一下缓冲队列，重新记录数据
+                    binlogParser.reset();// 重新置位
+
+                    if (running) {
+                        // sleep一段时间再进行重试
+                        try {
+                            Thread.sleep(10000 + RandomUtils.nextInt(10000));
+                        } catch (InterruptedException e) {
+                        }
+                    }
+                }
+                MDC.remove("destination");
+            }
+        });
+
+        parseThread.setUncaughtExceptionHandler(handler);
+        parseThread.setName(String.format("destination = %s , address = %s , EventParser",
+            destination,
+            runningInfo == null ? null : runningInfo.getAddress().toString()));
+        parseThread.start();
+    }
+
+    public void stop() {
+        super.stop();
+
+        stopHeartBeat(); // 先停止心跳
+        parseThread.interrupt(); // 尝试中断
+        eventSink.interrupt();
+        try {
+            parseThread.join();// 等待其结束
+        } catch (InterruptedException e) {
+            // ignore
+        }
+
+        if (binlogParser.isStart()) {
+            binlogParser.stop();
+        }
+        if (transactionBuffer.isStart()) {
+            transactionBuffer.stop();
+        }
+    }
+
+    protected boolean consumeTheEventAndProfilingIfNecessary(List entrys) throws CanalSinkException,
+                                                                                           InterruptedException {
+        long startTs = -1;
+        boolean enabled = getProfilingEnabled();
+        if (enabled) {
+            startTs = System.currentTimeMillis();
+        }
+
+        boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);
+
+        if (enabled) {
+            this.processingInterval = System.currentTimeMillis() - startTs;
+        }
+
+        if (consumedEventCount.incrementAndGet() < 0) {
+            consumedEventCount.set(0);
+        }
+
+        return result;
+    }
+
+    protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception {
+        long startTs = -1;
+        boolean enabled = getProfilingEnabled();
+        if (enabled) {
+            startTs = System.currentTimeMillis();
+        }
+        CanalEntry.Entry event = binlogParser.parse(bod);
+
+        if (enabled) {
+            this.parsingInterval = System.currentTimeMillis() - startTs;
+        }
+
+        if (parsedEventCount.incrementAndGet() < 0) {
+            parsedEventCount.set(0);
+        }
+        return event;
+    }
+
+    public Boolean getProfilingEnabled() {
+        return profilingEnabled.get();
+    }
+
+    protected LogPosition buildLastTranasctionPosition(List entries) { // 初始化一下
+        for (int i = entries.size() - 1; i > 0; i--) {
+            CanalEntry.Entry entry = entries.get(i);
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
+                LogPosition logPosition = new LogPosition();
+                EntryPosition position = new EntryPosition();
+                position.setJournalName(entry.getHeader().getLogfileName());
+                position.setPosition(entry.getHeader().getLogfileOffset());
+                position.setTimestamp(entry.getHeader().getExecuteTime());
+                logPosition.setPostion(position);
+
+                LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
+                logPosition.setIdentity(identity);
+                return logPosition;
+            }
+        }
+
+        return null;
+    }
+
+    protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
+        LogPosition logPosition = new LogPosition();
+        EntryPosition position = new EntryPosition();
+        position.setJournalName(entry.getHeader().getLogfileName());
+        position.setPosition(entry.getHeader().getLogfileOffset());
+        position.setTimestamp(entry.getHeader().getExecuteTime());
+        logPosition.setPostion(position);
+
+        LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
+        logPosition.setIdentity(identity);
+        return logPosition;
+    }
+
+    protected void processError(Exception e, LogPosition lastPosition, String startBinlogFile, long startPosition) {
+        if (lastPosition != null) {
+            logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
+                lastPosition.getPostion()),
+                e);
+        } else {
+            logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s,%s]",
+                startBinlogFile,
+                startPosition), e);
+        }
+    }
+
+    protected void startHeartBeat(ErosaConnection connection) {
+        lastEntryTime = 0L; // 初始化
+        if (timer == null) {// lazy初始化一下
+            String name = String.format("destination = %s , address = %s , HeartBeatTimeTask",
+                destination,
+                runningInfo == null ? null : runningInfo.getAddress().toString());
+            synchronized (MysqlEventParser.class) {
+                if (timer == null) {
+                    timer = new Timer(name, true);
+                }
+            }
+        }
+
+        if (heartBeatTimerTask == null) {// fixed issue #56，避免重复创建heartbeat线程
+            heartBeatTimerTask = buildHeartBeatTimeTask(connection);
+            Integer interval = detectingIntervalInSeconds;
+            timer.schedule(heartBeatTimerTask, interval * 1000L, interval * 1000L);
+            logger.info("start heart beat.... ");
+        }
+    }
+
+    protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
+        return new TimerTask() {
+
+            public void run() {
+                try {
+                    if (exception == null || lastEntryTime > 0) {
+                        // 如果未出现异常，或者有第一条正常数据
+                        long now = System.currentTimeMillis();
+                        long inteval = (now - lastEntryTime) / 1000;
+                        if (inteval >= detectingIntervalInSeconds) {
+                            Header.Builder headerBuilder = Header.newBuilder();
+                            headerBuilder.setExecuteTime(now);
+                            Entry.Builder entryBuilder = Entry.newBuilder();
+                            entryBuilder.setHeader(headerBuilder.build());
+                            entryBuilder.setEntryType(EntryType.HEARTBEAT);
+                            Entry entry = entryBuilder.build();
+                            // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
+                            consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
+                        }
+                    }
+
+                } catch (Throwable e) {
+                    logger.warn("heartBeat run failed " + ExceptionUtils.getStackTrace(e));
+                }
+            }
+
+        };
+    }
+
+    protected void stopHeartBeat() {
+        lastEntryTime = 0L; // 初始化
+        if (timer != null) {
+            timer.cancel();
+            timer = null;
+        }
+        heartBeatTimerTask = null;
+    }
+
+    public void setEventFilter(CanalEventFilter eventFilter) {
+        this.eventFilter = eventFilter;
+    }
+
+    public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
+        this.eventBlackFilter = eventBlackFilter;
+    }
+
+    public Long getParsedEventCount() {
+        return parsedEventCount.get();
+    }
+
+    public Long getConsumedEventCount() {
+        return consumedEventCount.get();
+    }
+
+    public void setProfilingEnabled(boolean profilingEnabled) {
+        this.profilingEnabled = new AtomicBoolean(profilingEnabled);
+    }
+
+    public long getParsingInterval() {
+        return parsingInterval;
+    }
+
+    public long getProcessingInterval() {
+        return processingInterval;
+    }
+
+    public void setEventSink(CanalEventSink> eventSink) {
+        this.eventSink = eventSink;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void setBinlogParser(BinlogParser binlogParser) {
+        this.binlogParser = binlogParser;
+    }
+
+    public BinlogParser getBinlogParser() {
+        return binlogParser;
+    }
+
+    public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
+        this.alarmHandler = alarmHandler;
+    }
+
+    public CanalAlarmHandler getAlarmHandler() {
+        return this.alarmHandler;
+    }
+
+    public void setLogPositionManager(CanalLogPositionManager logPositionManager) {
+        this.logPositionManager = logPositionManager;
+    }
+
+    public void setTransactionSize(int transactionSize) {
+        this.transactionSize = transactionSize;
+    }
+
+    public CanalLogPositionManager getLogPositionManager() {
+        return logPositionManager;
+    }
+
+    public void setDetectingEnable(boolean detectingEnable) {
+        this.detectingEnable = detectingEnable;
+    }
+
+    public void setDetectingIntervalInSeconds(Integer detectingIntervalInSeconds) {
+        this.detectingIntervalInSeconds = detectingIntervalInSeconds;
+    }
+
+    public Throwable getException() {
+        return exception;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java
new file mode 100644
index 00000000..3ff91287
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/BinlogParser.java
@@ -0,0 +1,17 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+
+/**
+ * 解析binlog的接口
+ * 
+ * @author: yuanzu Date: 12-9-20 Time: 下午8:46
+ */
+public interface BinlogParser extends CanalLifeCycle {
+
+    CanalEntry.Entry parse(T event) throws CanalParseException;
+
+    void reset();
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
new file mode 100644
index 00000000..4b32bb94
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/ErosaConnection.java
@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.io.IOException;
+
+/**
+ * 通用的Erosa的链接接口, 用于一般化处理mysql/oracle的解析过程
+ * 
+ * @author: yuanzu Date: 12-9-20 Time: 下午2:47
+ */
+public interface ErosaConnection {
+
+    public void connect() throws IOException;
+
+    public void reconnect() throws IOException;
+
+    public void disconnect() throws IOException;
+
+    public boolean isConnected();
+
+    /**
+     * 用于快速数据查找,和dump的区别在于，seek会只给出部分的数据
+     */
+    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
+
+    public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException;
+
+    public void dump(long timestamp, SinkFunction func) throws IOException;
+
+    ErosaConnection fork();
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java
new file mode 100644
index 00000000..c3c7150f
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBuffer.java
@@ -0,0 +1,164 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.store.CanalStoreException;
+
+/**
+ * 缓冲event队列，提供按事务刷新数据的机制
+ * 
+ * @author jianghang 2012-12-6 上午11:05:12
+ * @version 1.0.0
+ */
+public class EventTransactionBuffer extends AbstractCanalLifeCycle {
+
+    private static final long        INIT_SQEUENCE = -1;
+    private int                      bufferSize    = 1024;
+    private int                      indexMask;
+    private CanalEntry.Entry[]       entries;
+
+    private AtomicLong               putSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前put操作最后一次写操作发生的位置
+    private AtomicLong               flushSequence = new AtomicLong(INIT_SQEUENCE); // 代表满足flush条件后最后一次数据flush的时间
+
+    private TransactionFlushCallback flushCallback;
+
+    public EventTransactionBuffer(){
+
+    }
+
+    public EventTransactionBuffer(TransactionFlushCallback flushCallback){
+        this.flushCallback = flushCallback;
+    }
+
+    public void start() throws CanalStoreException {
+        super.start();
+        if (Integer.bitCount(bufferSize) != 1) {
+            throw new IllegalArgumentException("bufferSize must be a power of 2");
+        }
+
+        Assert.notNull(flushCallback, "flush callback is null!");
+        indexMask = bufferSize - 1;
+        entries = new CanalEntry.Entry[bufferSize];
+    }
+
+    public void stop() throws CanalStoreException {
+        putSequence.set(INIT_SQEUENCE);
+        flushSequence.set(INIT_SQEUENCE);
+
+        entries = null;
+        super.stop();
+    }
+
+    public void add(List entrys) throws InterruptedException {
+        for (CanalEntry.Entry entry : entrys) {
+            add(entry);
+        }
+    }
+
+    public void add(CanalEntry.Entry entry) throws InterruptedException {
+        switch (entry.getEntryType()) {
+            case TRANSACTIONBEGIN:
+                flush();// 刷新上一次的数据
+                put(entry);
+                break;
+            case TRANSACTIONEND:
+                put(entry);
+                flush();
+                break;
+            case ROWDATA:
+                put(entry);
+                // 针对非DML的数据，直接输出，不进行buffer控制
+                EventType eventType = entry.getHeader().getEventType();
+                if (eventType != null && !isDml(eventType)) {
+                    flush();
+                }
+                break;
+            default:
+                break;
+        }
+    }
+
+    public void reset() {
+        putSequence.set(INIT_SQEUENCE);
+        flushSequence.set(INIT_SQEUENCE);
+    }
+
+    private void put(CanalEntry.Entry data) throws InterruptedException {
+        // 首先检查是否有空位
+        if (checkFreeSlotAt(putSequence.get() + 1)) {
+            long current = putSequence.get();
+            long next = current + 1;
+
+            // 先写数据，再更新对应的cursor,并发度高的情况，putSequence会被get请求可见，拿出了ringbuffer中的老的Entry值
+            entries[getIndex(next)] = data;
+            putSequence.set(next);
+        } else {
+            flush();// buffer区满了，刷新一下
+            put(data);// 继续加一下新数据
+        }
+    }
+
+    private void flush() throws InterruptedException {
+        long start = this.flushSequence.get() + 1;
+        long end = this.putSequence.get();
+
+        if (start <= end) {
+            List transaction = new ArrayList();
+            for (long next = start; next <= end; next++) {
+                transaction.add(this.entries[getIndex(next)]);
+            }
+
+            flushCallback.flush(transaction);
+            flushSequence.set(end);// flush成功后，更新flush位置
+        }
+    }
+
+    /**
+     * 查询是否有空位
+     */
+    private boolean checkFreeSlotAt(final long sequence) {
+        final long wrapPoint = sequence - bufferSize;
+        if (wrapPoint > flushSequence.get()) { // 刚好追上一轮
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    private int getIndex(long sequcnce) {
+        return (int) sequcnce & indexMask;
+    }
+
+    private boolean isDml(EventType eventType) {
+        return eventType == EventType.INSERT || eventType == EventType.UPDATE || eventType == EventType.DELETE;
+    }
+
+    // ================ setter / getter ==================
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+    public void setFlushCallback(TransactionFlushCallback flushCallback) {
+        this.flushCallback = flushCallback;
+    }
+
+    /**
+     * 事务刷新机制
+     * 
+     * @author jianghang 2012-12-6 上午11:57:38
+     * @version 1.0.0
+     */
+    public static interface TransactionFlushCallback {
+
+        public void flush(List transaction) throws InterruptedException;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/HeartBeatCallback.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/HeartBeatCallback.java
new file mode 100644
index 00000000..d450df77
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/HeartBeatCallback.java
@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+/**
+ * 提供mysql heartBeat心跳数据的callback机制
+ * 
+ * @author jianghang 2012-6-26 下午04:49:56
+ * @version 1.0.0
+ */
+public interface HeartBeatCallback {
+
+    /**
+     * 心跳发送成功
+     */
+    public void onSuccess(long costTime);
+
+    /**
+     * 心跳发送失败
+     */
+    public void onFailed(Throwable e);
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/SinkFunction.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/SinkFunction.java
new file mode 100644
index 00000000..de9c0fac
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/SinkFunction.java
@@ -0,0 +1,12 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+/**
+ * receive parsed bytes , 用于处理要解析的数据块
+ * 
+ * @author: yuanzu Date: 12-9-20 Time: 下午2:50
+ */
+
+public interface SinkFunction {
+
+    public boolean sink(EVENT event);
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java
new file mode 100644
index 00000000..712919de
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/TableMeta.java
@@ -0,0 +1,122 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+
+/**
+ * 描述数据meta对象,mysql binlog中对应的{@linkplain TableMapLogEvent}包含的信息不全
+ * 
+ * 
+ * 1. 主键信息
+ * 2. column name
+ * 3. unsigned字段
+ * 

+ * 
+ * @author jianghang 2013-1-18 下午12:24:59
+ * @version 1.0.0
+ */
+public class TableMeta {
+
+    private String          fullName; // schema.table
+    private List fileds;
+
+    public TableMeta(String fullName, List fileds){
+        this.fullName = fullName;
+        this.fileds = fileds;
+    }
+
+    public String getFullName() {
+        return fullName;
+    }
+
+    public void setFullName(String fullName) {
+        this.fullName = fullName;
+    }
+
+    public List getFileds() {
+        return fileds;
+    }
+
+    public void setFileds(List fileds) {
+        this.fileds = fileds;
+    }
+
+    public static class FieldMeta {
+
+        private String columnName;
+        private String columnType;
+        private String isNullable;
+        private String iskey;
+        private String defaultValue;
+        private String extra;
+
+        public String getColumnName() {
+            return columnName;
+        }
+
+        public void setColumnName(String columnName) {
+            this.columnName = columnName;
+        }
+
+        public String getColumnType() {
+            return columnType;
+        }
+
+        public void setColumnType(String columnType) {
+            this.columnType = columnType;
+        }
+
+        public String getIsNullable() {
+            return isNullable;
+        }
+
+        public void setIsNullable(String isNullable) {
+            this.isNullable = isNullable;
+        }
+
+        public String getIskey() {
+            return iskey;
+        }
+
+        public void setIskey(String iskey) {
+            this.iskey = iskey;
+        }
+
+        public String getDefaultValue() {
+            return defaultValue;
+        }
+
+        public void setDefaultValue(String defaultValue) {
+            this.defaultValue = defaultValue;
+        }
+
+        public String getExtra() {
+            return extra;
+        }
+
+        public void setExtra(String extra) {
+            this.extra = extra;
+        }
+
+        public boolean isUnsigned() {
+            return StringUtils.containsIgnoreCase(columnType, "unsigned");
+        }
+
+        public boolean isKey() {
+            return StringUtils.equalsIgnoreCase(iskey, "PRI");
+        }
+
+        public boolean isNullable() {
+            return StringUtils.equalsIgnoreCase(isNullable, "YES");
+        }
+
+        public String toString() {
+            return "FieldMeta [columnName=" + columnName + ", columnType=" + columnType + ", defaultValue="
+                   + defaultValue + ", extra=" + extra + ", isNullable=" + isNullable + ", iskey=" + iskey + "]";
+        }
+
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java
new file mode 100644
index 00000000..b467afc6
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventParser.java
@@ -0,0 +1,57 @@
+package com.alibaba.otter.canal.parse.inbound.group;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+
+/**
+ * 组合多个EventParser进行合并处理，group只是做为一个delegate处理
+ * 
+ * @author jianghang 2012-10-16 上午11:23:14
+ * @version 1.0.0
+ */
+public class GroupEventParser extends AbstractCanalLifeCycle implements CanalEventParser {
+
+    private List eventParsers = new ArrayList();
+
+    public void start() {
+        super.start();
+        // 统一启动
+        for (CanalEventParser eventParser : eventParsers) {
+            if (!eventParser.isStart()) {
+                eventParser.start();
+            }
+        }
+    }
+
+    public void stop() {
+        super.stop();
+        // 统一关闭
+        for (CanalEventParser eventParser : eventParsers) {
+            if (eventParser.isStart()) {
+                eventParser.stop();
+            }
+        }
+    }
+
+    public void setEventParsers(List eventParsers) {
+        this.eventParsers = eventParsers;
+    }
+
+    public void addEventParser(CanalEventParser eventParser) {
+        if (!eventParsers.contains(eventParser)) {
+            eventParsers.add(eventParser);
+        }
+    }
+
+    public void removeEventParser(CanalEventParser eventParser) {
+        eventParsers.remove(eventParser);
+    }
+
+    public List getEventParsers() {
+        return eventParsers;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
new file mode 100644
index 00000000..85644dcd
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/AbstractMysqlEventParser.java
@@ -0,0 +1,94 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.nio.charset.Charset;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.parse.inbound.AbstractEventParser;
+import com.alibaba.otter.canal.parse.inbound.BinlogParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
+
+public abstract class AbstractMysqlEventParser extends AbstractEventParser {
+
+    protected final Logger      logger                  = LoggerFactory.getLogger(this.getClass());
+    protected static final long BINLOG_START_OFFEST     = 4L;
+
+    // 编码信息
+    protected byte              connectionCharsetNumber = (byte) 33;
+    protected Charset           connectionCharset       = Charset.forName("UTF-8");
+    protected boolean           filterQueryDcl          = false;
+    protected boolean           filterQueryDml          = false;
+    protected boolean           filterQueryDdl          = false;
+    protected boolean           filterTableError        = false;
+
+    protected BinlogParser buildParser() {
+        LogEventConvert convert = new LogEventConvert();
+        if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
+            convert.setNameFilter((AviaterRegexFilter) eventFilter);
+        }
+
+        if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
+            convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
+        }
+
+        convert.setCharset(connectionCharset);
+        convert.setFilterQueryDcl(filterQueryDcl);
+        convert.setFilterQueryDml(filterQueryDml);
+        convert.setFilterQueryDdl(filterQueryDdl);
+        convert.setFilterTableError(filterTableError);
+        return convert;
+    }
+
+    public void setEventFilter(CanalEventFilter eventFilter) {
+        super.setEventFilter(eventFilter);
+
+        // 触发一下filter变更
+        if (eventFilter != null && eventFilter instanceof AviaterRegexFilter && binlogParser instanceof LogEventConvert) {
+            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
+        }
+    }
+
+    public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
+        super.setEventBlackFilter(eventBlackFilter);
+
+        // 触发一下filter变更
+        if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter
+            && binlogParser instanceof LogEventConvert) {
+            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
+        }
+    }
+
+    // ============================ setter / getter =========================
+
+    public void setConnectionCharsetNumber(byte connectionCharsetNumber) {
+        this.connectionCharsetNumber = connectionCharsetNumber;
+    }
+
+    public void setConnectionCharset(Charset connectionCharset) {
+        this.connectionCharset = connectionCharset;
+    }
+
+    public void setConnectionCharset(String connectionCharset) {
+        this.connectionCharset = Charset.forName(connectionCharset);
+    }
+
+    public void setFilterQueryDcl(boolean filterQueryDcl) {
+        this.filterQueryDcl = filterQueryDcl;
+    }
+
+    public void setFilterQueryDml(boolean filterQueryDml) {
+        this.filterQueryDml = filterQueryDml;
+    }
+
+    public void setFilterQueryDdl(boolean filterQueryDdl) {
+        this.filterQueryDdl = filterQueryDdl;
+    }
+
+    public void setFilterTableError(boolean filterTableError) {
+        this.filterTableError = filterTableError;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/DbsyncMysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/DbsyncMysqlEventParser.java
new file mode 100644
index 00000000..b8501ffe
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/DbsyncMysqlEventParser.java
@@ -0,0 +1,5 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+public class DbsyncMysqlEventParser {
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
new file mode 100644
index 00000000..dc821ce4
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinLogConnection.java
@@ -0,0 +1,244 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.SinkFunction;
+import com.alibaba.otter.canal.parse.inbound.mysql.local.BinLogFileQueue;
+import com.taobao.tddl.dbsync.binlog.FileLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.LogPosition;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+
+/**
+ * local bin log connection (not real connection)
+ * 
+ * @author yuanzu Date: 12-9-27 Time: 下午6:14
+ */
+public class LocalBinLogConnection implements ErosaConnection {
+
+    private static final Logger logger     = LoggerFactory.getLogger(LocalBinLogConnection.class);
+    private BinLogFileQueue     binlogs    = null;
+    private boolean             needWait;
+    private String              directory;
+    private int                 bufferSize = 16 * 1024;
+    private boolean             running    = false;
+
+    public LocalBinLogConnection(){
+    }
+
+    public LocalBinLogConnection(String directory, boolean needWait){
+        this.needWait = needWait;
+        this.directory = directory;
+    }
+
+    @Override
+    public void connect() throws IOException {
+        if (this.binlogs == null) {
+            this.binlogs = new BinLogFileQueue(this.directory);
+        }
+        this.running = true;
+    }
+
+    @Override
+    public void reconnect() throws IOException {
+        disconnect();
+        connect();
+    }
+
+    @Override
+    public void disconnect() throws IOException {
+        this.running = false;
+        if (this.binlogs != null) {
+            this.binlogs.destory();
+        }
+        this.binlogs = null;
+        this.running = false;
+    }
+
+    public boolean isConnected() {
+        return running;
+    }
+
+    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+    }
+
+    public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+        File current = new File(directory, binlogfilename);
+
+        FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
+        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+        LogContext context = new LogContext();
+        try {
+            fetcher.open(current, binlogPosition);
+            context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
+            while (running) {
+                boolean needContinue = true;
+                LogEvent event;
+                while (fetcher.fetch()) {
+                    event = decoder.decode(fetcher, context);
+                    if (event == null) {
+                        throw new CanalParseException("parse failed");
+                    }
+
+                    if (!func.sink(event)) {
+                        needContinue = false;
+                        break;
+                    }
+
+                    // do {
+                    // event = decoder.decode(fetcher, context);
+                    // if (event != null && !func.sink(event)) {
+                    // needContinue = false;
+                    // break;
+                    // }
+                    // } while (event != null);
+                }
+
+                if (needContinue) {// 读取下一个
+                    fetcher.close(); // 关闭上一个文件
+
+                    File nextFile;
+                    if (needWait) {
+                        nextFile = binlogs.waitForNextFile(current);
+                    } else {
+                        nextFile = binlogs.getNextFile(current);
+                    }
+
+                    if (nextFile == null) {
+                        break;
+                    }
+
+                    current = nextFile;
+
+                    fetcher.open(current);
+                    context.setLogPosition(new LogPosition(nextFile.getName()));
+                } else {
+                    break;// 跳出
+                }
+            }
+        } catch (InterruptedException e) {
+            logger.warn("LocalBinLogConnection dump interrupted");
+        } finally {
+            if (fetcher != null) {
+                fetcher.close();
+            }
+        }
+    }
+
+    public void dump(long timestampMills, SinkFunction func) throws IOException {
+        List currentBinlogs = binlogs.currentBinlogs();
+        File current = currentBinlogs.get(currentBinlogs.size() - 1);
+        long timestampSeconds = timestampMills / 1000;
+
+        String binlogFilename = null;
+        long binlogFileOffset = 0;
+
+        FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
+        LogDecoder decoder = new LogDecoder();
+        decoder.handle(LogEvent.QUERY_EVENT);
+        decoder.handle(LogEvent.XID_EVENT);
+        LogContext context = new LogContext();
+        try {
+            fetcher.open(current);
+            context.setLogPosition(new LogPosition(current.getName()));
+            while (running) {
+                boolean needContinue = true;
+                String lastXidLogFilename = current.getName();
+                long lastXidLogFileOffset = 0;
+
+                binlogFilename = lastXidLogFilename;
+                binlogFileOffset = lastXidLogFileOffset;
+                L: while (fetcher.fetch()) {
+                    LogEvent event;
+                    do {
+                        event = decoder.decode(fetcher, context);
+                        if (event != null) {
+                            if (event.getWhen() > timestampSeconds) {
+                                break L;
+                            }
+
+                            needContinue = false;
+                            if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
+                                if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
+                                    binlogFilename = lastXidLogFilename;
+                                    binlogFileOffset = lastXidLogFileOffset;
+                                } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
+                                    lastXidLogFilename = current.getName();
+                                    lastXidLogFileOffset = event.getLogPos();
+                                }
+                            } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
+                                lastXidLogFilename = current.getName();
+                                lastXidLogFileOffset = event.getLogPos();
+                            }
+                        }
+                    } while (event != null);
+                }
+
+                if (needContinue) {// 读取下一个
+                    fetcher.close(); // 关闭上一个文件
+
+                    File nextFile = binlogs.getBefore(current);
+                    if (nextFile == null) {
+                        break;
+                    }
+
+                    current = nextFile;
+                    fetcher.open(current);
+                    context.setLogPosition(new LogPosition(current.getName()));
+                } else {
+                    break;// 跳出
+                }
+            }
+        } finally {
+            if (fetcher != null) {
+                fetcher.close();
+            }
+        }
+
+        dump(binlogFilename, binlogFileOffset, func);
+    }
+
+    public ErosaConnection fork() {
+        LocalBinLogConnection connection = new LocalBinLogConnection();
+
+        connection.setBufferSize(this.bufferSize);
+        connection.setDirectory(this.directory);
+        connection.setNeedWait(this.needWait);
+        return connection;
+    }
+
+    public boolean isNeedWait() {
+        return needWait;
+    }
+
+    public void setNeedWait(boolean needWait) {
+        this.needWait = needWait;
+    }
+
+    public String getDirectory() {
+        return directory;
+    }
+
+    public void setDirectory(String directory) {
+        this.directory = directory;
+    }
+
+    public int getBufferSize() {
+        return bufferSize;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
new file mode 100644
index 00000000..2317c0c2
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParser.java
@@ -0,0 +1,118 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 基于本地binlog文件的复制
+ * 
+ * @author jianghang 2012-6-21 下午04:07:33
+ * @version 1.0.0
+ */
+public class LocalBinlogEventParser extends AbstractMysqlEventParser implements CanalEventParser {
+
+    // 数据库信息
+    private AuthenticationInfo masterInfo;
+    private EntryPosition      masterPosition;        // binlog信息
+
+    private String             directory;
+    private boolean            needWait   = false;
+    private int                bufferSize = 16 * 1024;
+
+    public LocalBinlogEventParser(){
+        // this.runningInfo = new AuthenticationInfo();
+    }
+
+    @Override
+    protected ErosaConnection buildErosaConnection() {
+        return buildLocalBinLogConnection();
+    }
+
+    public void start() throws CanalParseException {
+        if (runningInfo == null) { // 第一次链接主库
+            runningInfo = masterInfo;
+        }
+
+        super.start();
+    }
+
+    private ErosaConnection buildLocalBinLogConnection() {
+        LocalBinLogConnection connection = new LocalBinLogConnection();
+
+        connection.setBufferSize(this.bufferSize);
+        connection.setDirectory(this.directory);
+        connection.setNeedWait(this.needWait);
+
+        return connection;
+    }
+
+    @Override
+    protected EntryPosition findStartPosition(ErosaConnection connection) {
+        // 处理逻辑
+        // 1. 首先查询上一次解析成功的最后一条记录
+        // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
+        // // a. 无机器切换，直接返回
+        // // b. 存在机器切换，按最后一条记录的stamptime进行查找
+        // 3. 不存在最后一条记录，则从默认的位置开始启动
+        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+        if (logPosition == null) {// 找不到历史成功记录
+            EntryPosition entryPosition = masterPosition;
+
+            // 判断一下是否需要按时间订阅
+            if (StringUtils.isEmpty(entryPosition.getJournalName())) {
+                // 如果没有指定binlogName，尝试按照timestamp进行查找
+                if (entryPosition.getTimestamp() != null) {
+                    return new EntryPosition(entryPosition.getTimestamp());
+                }
+            } else {
+                if (entryPosition.getPosition() != null) {
+                    // 如果指定binlogName + offest，直接返回
+                    return entryPosition;
+                } else {
+                    return new EntryPosition(entryPosition.getTimestamp());
+                }
+            }
+        } else {
+            return logPosition.getPostion();
+        }
+
+        return null;
+    }
+
+    // ========================= setter / getter =========================
+
+    public void setLogPositionManager(CanalLogPositionManager logPositionManager) {
+        this.logPositionManager = logPositionManager;
+    }
+
+    public void setDirectory(String directory) {
+        this.directory = directory;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+    public void setMasterPosition(EntryPosition masterPosition) {
+        this.masterPosition = masterPosition;
+    }
+
+    public void setMasterInfo(AuthenticationInfo masterInfo) {
+        this.masterInfo = masterInfo;
+    }
+
+    public boolean isNeedWait() {
+        return needWait;
+    }
+
+    public void setNeedWait(boolean needWait) {
+        this.needWait = needWait;
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
new file mode 100644
index 00000000..840b9875
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
@@ -0,0 +1,306 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.SinkFunction;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class MysqlConnection implements ErosaConnection {
+
+    private static final Logger logger       = LoggerFactory.getLogger(MysqlConnection.class);
+
+    private MysqlConnector      connector;
+    private long                slaveId;
+    private Charset             charset      = Charset.forName("UTF-8");
+    private BinlogFormat        binlogFormat = BinlogFormat.ROW;
+
+    public MysqlConnection(){
+    }
+
+    public MysqlConnection(InetSocketAddress address, String username, String password){
+
+        connector = new MysqlConnector(address, username, password);
+    }
+
+    public MysqlConnection(InetSocketAddress address, String username, String password, byte charsetNumber,
+                           String defaultSchema){
+        connector = new MysqlConnector(address, username, password, charsetNumber, defaultSchema);
+    }
+
+    public void connect() throws IOException {
+        connector.connect();
+    }
+
+    public void reconnect() throws IOException {
+        connector.reconnect();
+    }
+
+    public void disconnect() throws IOException {
+        connector.disconnect();
+    }
+
+    public boolean isConnected() {
+        return connector.isConnected();
+    }
+
+    public ResultSetPacket query(String cmd) throws IOException {
+        MysqlQueryExecutor exector = new MysqlQueryExecutor(connector);
+        return exector.query(cmd);
+    }
+
+    public void update(String cmd) throws IOException {
+        MysqlUpdateExecutor exector = new MysqlUpdateExecutor(connector);
+        exector.update(cmd);
+    }
+
+    /**
+     * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
+     */
+    public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+        updateSettings();
+
+        sendBinlogDump(binlogfilename, binlogPosition);
+        DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
+        fetcher.start(connector.getChannel());
+        LogDecoder decoder = new LogDecoder();
+        decoder.handle(LogEvent.ROTATE_EVENT);
+        decoder.handle(LogEvent.QUERY_EVENT);
+        decoder.handle(LogEvent.XID_EVENT);
+        LogContext context = new LogContext();
+        while (fetcher.fetch()) {
+            LogEvent event = null;
+            event = decoder.decode(fetcher, context);
+
+            if (event == null) {
+                throw new CanalParseException("parse failed");
+            }
+
+            if (!func.sink(event)) {
+                break;
+            }
+        }
+    }
+
+    public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
+        updateSettings();
+        BinlogFormat format = getBinlogFormat();
+        if (!format.isRow()) {
+            logger.warn("binlog_format : {} , it will not have rowData before/after columns", format.toString());
+        }
+
+        sendBinlogDump(binlogfilename, binlogPosition);
+        DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
+        fetcher.start(connector.getChannel());
+        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+        LogContext context = new LogContext();
+        while (fetcher.fetch()) {
+            LogEvent event = null;
+            event = decoder.decode(fetcher, context);
+
+            if (event == null) {
+                throw new CanalParseException("parse failed");
+            }
+
+            if (!func.sink(event)) {
+                break;
+            }
+        }
+    }
+
+    public void dump(long timestamp, SinkFunction func) throws IOException {
+        throw new NullPointerException("Not implement yet");
+    }
+
+    private void sendBinlogDump(String binlogfilename, Long binlogPosition) throws IOException {
+        BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
+        binlogDumpCmd.binlogFileName = binlogfilename;
+        binlogDumpCmd.binlogPosition = binlogPosition;
+        binlogDumpCmd.slaveServerId = this.slaveId;
+        byte[] cmdBody = binlogDumpCmd.toBytes();
+
+        logger.info("COM_BINLOG_DUMP with position:{}", binlogDumpCmd);
+        HeaderPacket binlogDumpHeader = new HeaderPacket();
+        binlogDumpHeader.setPacketBodyLength(cmdBody.length);
+        binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
+        PacketManager.write(connector.getChannel(), new ByteBuffer[] { ByteBuffer.wrap(binlogDumpHeader.toBytes()),
+                ByteBuffer.wrap(cmdBody) });
+    }
+
+    public MysqlConnection fork() {
+        MysqlConnection connection = new MysqlConnection();
+        connection.setCharset(getCharset());
+        connection.setSlaveId(getSlaveId());
+        connection.setConnector(connector.fork());
+        return connection;
+    }
+
+    // ====================== help method ====================
+
+    /**
+     * the settings that will need to be checked or set:

+     * 
    
+     * 
  1. wait_timeout
    2. 
+     * 
  2. net_write_timeout
    3. 
+     * 
  3. net_read_timeout
    4. 
+     * 

+     * 
+     * @param channel
+     * @throws IOException
+     */
+    private void updateSettings() throws IOException {
+        try {
+            update("set wait_timeout=9999999");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+        try {
+            update("set net_write_timeout=1800");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        try {
+            update("set net_read_timeout=1800");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        try {
+            // 设置服务端返回结果时不做编码转化，直接按照数据库的二进制编码进行发送，由客户端自己根据需求进行编码转化
+            update("set names 'binary'");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        try {
+            // mysql5.6针对checksum支持需要设置session变量
+            // 如果不设置会出现错误： Slave can not handle replication events with the
+            // checksum that master is configured to log
+            // 但也不能乱设置，需要和mysql server的checksum配置一致，不然RotateLogEvent会出现乱码
+            update("set @master_binlog_checksum= '@@global.binlog_checksum'");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+
+        
+        try {
+            // mariadb针对特殊的类型，需要设置session变量
+            update("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
+        } catch (Exception e) {
+            logger.warn(ExceptionUtils.getFullStackTrace(e));
+        }
+    }
+
+    /**
+     * 判断一下是否采用ROW模式
+     */
+    private void loadBinlogFormat() {
+        ResultSetPacket rs = null;
+        try {
+            rs = query("show variables like 'binlog_format'");
+        } catch (IOException e) {
+            throw new CanalParseException(e);
+        }
+
+        List columnValues = rs.getFieldValues();
+        if (columnValues == null || columnValues.size() != 2) {
+            logger.warn("unexpected binlog format query result, this may cause unexpected result, so throw exception to request network to io shutdown.");
+            throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
+        }
+
+        binlogFormat = BinlogFormat.valuesOf(columnValues.get(1));
+        if (binlogFormat == null) {
+            throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
+        }
+    }
+
+    public static enum BinlogFormat {
+
+        STATEMENT("STATEMENT"), ROW("ROW"), MIXED("MIXED");
+
+        public boolean isStatement() {
+            return this == STATEMENT;
+        }
+
+        public boolean isRow() {
+            return this == ROW;
+        }
+
+        public boolean isMixed() {
+            return this == MIXED;
+        }
+
+        private String value;
+
+        private BinlogFormat(String value){
+            this.value = value;
+        }
+
+        public static BinlogFormat valuesOf(String value) {
+            BinlogFormat[] formats = values();
+            for (BinlogFormat format : formats) {
+                if (format.value.equalsIgnoreCase(value)) {
+                    return format;
+                }
+            }
+            return null;
+        }
+    }
+
+    // ================== setter / getter ===================
+
+    public Charset getCharset() {
+        return charset;
+    }
+
+    public void setCharset(Charset charset) {
+        this.charset = charset;
+    }
+
+    public long getSlaveId() {
+        return slaveId;
+    }
+
+    public void setSlaveId(long slaveId) {
+        this.slaveId = slaveId;
+    }
+
+    public MysqlConnector getConnector() {
+        return connector;
+    }
+
+    public void setConnector(MysqlConnector connector) {
+        this.connector = connector;
+    }
+
+    public BinlogFormat getBinlogFormat() {
+        if (binlogFormat == null) {
+            synchronized (this) {
+                loadBinlogFormat();
+            }
+        }
+
+        return binlogFormat;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
new file mode 100644
index 00000000..d81d5ded
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
@@ -0,0 +1,738 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.CanalHASwitchable;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.ha.CanalHAController;
+import com.alibaba.otter.canal.parse.inbound.ErosaConnection;
+import com.alibaba.otter.canal.parse.inbound.HeartBeatCallback;
+import com.alibaba.otter.canal.parse.inbound.SinkFunction;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.LogEventConvert;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * 基于向mysql server复制binlog实现
+ * 
+ * 
+ * 1. 自身不控制mysql主备切换，由ha机制来控制. 比如接入tddl/cobar/自身心跳包成功率
+ * 2. 切换机制
+ * 

+ * 
+ * @author jianghang 2012-6-21 下午04:06:32
+ * @version 1.0.0
+ */
+public class MysqlEventParser extends AbstractMysqlEventParser implements CanalEventParser, CanalHASwitchable {
+
+    private CanalHAController  haController                      = null;
+
+    private int                defaultConnectionTimeoutInSeconds = 30;       // sotimeout
+    private int                receiveBufferSize                 = 64 * 1024;
+    private int                sendBufferSize                    = 64 * 1024;
+    // 数据库信息
+    private AuthenticationInfo masterInfo;                                   // 主库
+    private AuthenticationInfo standbyInfo;                                  // 备库
+    // binlog信息
+    private EntryPosition      masterPosition;
+    private EntryPosition      standbyPosition;
+    private long               slaveId;                                      // 链接到mysql的slave
+    // 心跳检查信息
+    private String             detectingSQL;                                 // 心跳sql
+    private MysqlConnection    metaConnection;                               // 查询meta信息的链接
+    private TableMetaCache     tableMetaCache;                               // 对应meta
+                                                                              // cache
+    private int                fallbackIntervalInSeconds         = 60;       // 切换回退时间
+
+    // 心跳检查
+
+    protected ErosaConnection buildErosaConnection() {
+        return buildMysqlConnection(this.runningInfo);
+    }
+
+    protected void preDump(ErosaConnection connection) {
+        if (!(connection instanceof MysqlConnection)) {
+            throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
+        }
+
+        if (binlogParser != null && binlogParser instanceof LogEventConvert) {
+            metaConnection = (MysqlConnection) connection.fork();
+            try {
+                metaConnection.connect();
+            } catch (IOException e) {
+                throw new CanalParseException(e);
+            }
+
+            tableMetaCache = new TableMetaCache(metaConnection);
+            ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
+        }
+    }
+
+    protected void afterDump(ErosaConnection connection) {
+        super.afterDump(connection);
+
+        if (!(connection instanceof MysqlConnection)) {
+            throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
+        }
+
+        if (metaConnection != null) {
+            try {
+                metaConnection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+    }
+
+    public void start() throws CanalParseException {
+        if (runningInfo == null) { // 第一次链接主库
+            runningInfo = masterInfo;
+        }
+
+        super.start();
+    }
+
+    public void stop() throws CanalParseException {
+        if (metaConnection != null) {
+            try {
+                metaConnection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+
+        if (tableMetaCache != null) {
+            tableMetaCache.clearTableMeta();
+        }
+
+        super.stop();
+    }
+
+    protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
+        if (!(connection instanceof MysqlConnection)) {
+            throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
+        }
+
+        // 开始mysql心跳sql
+        if (detectingEnable && StringUtils.isNotBlank(detectingSQL)) {
+            return new MysqlDetectingTimeTask((MysqlConnection) connection.fork());
+        } else {
+            return super.buildHeartBeatTimeTask(connection);
+        }
+
+    }
+
+    protected void stopHeartBeat() {
+        super.stopHeartBeat();
+
+        if (heartBeatTimerTask != null) {
+
+            MysqlConnection mysqlConnection = ((MysqlDetectingTimeTask) heartBeatTimerTask).getMysqlConnection();
+            try {
+                mysqlConnection.disconnect();
+            } catch (IOException e) {
+                logger.error("ERROR # disconnect heartbeat connection for address:{}", mysqlConnection.getConnector()
+                    .getAddress(), e);
+            }
+        }
+    }
+
+    /**
+     * 心跳信息
+     * 
+     * @author jianghang 2012-7-6 下午02:50:15
+     * @version 1.0.0
+     */
+    class MysqlDetectingTimeTask extends TimerTask {
+
+        private boolean         reconnect = false;
+        private MysqlConnection mysqlConnection;
+
+        public MysqlDetectingTimeTask(MysqlConnection mysqlConnection){
+            this.mysqlConnection = mysqlConnection;
+        }
+
+        public void run() {
+            try {
+                if (reconnect) {
+                    reconnect = false;
+                    mysqlConnection.reconnect();
+                } else if (!mysqlConnection.isConnected()) {
+                    mysqlConnection.connect();
+                }
+                Long startTime = System.currentTimeMillis();
+
+                // 可能心跳sql为select 1
+                if (StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "select")
+                    || StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "show")
+                    || StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "explain")
+                    || StringUtils.startsWithIgnoreCase(detectingSQL.trim(), "desc")) {
+                    mysqlConnection.query(detectingSQL);
+                } else {
+                    mysqlConnection.update(detectingSQL);
+                }
+
+                Long costTime = System.currentTimeMillis() - startTime;
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onSuccess(costTime);
+                }
+            } catch (SocketTimeoutException e) {
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onFailed(e);
+                }
+                reconnect = true;
+                logger.warn("connect failed by " + ExceptionUtils.getStackTrace(e));
+            } catch (IOException e) {
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onFailed(e);
+                }
+                reconnect = true;
+                logger.warn("connect failed by " + ExceptionUtils.getStackTrace(e));
+            } catch (Throwable e) {
+                if (haController != null && haController instanceof HeartBeatCallback) {
+                    ((HeartBeatCallback) haController).onFailed(e);
+                }
+                reconnect = true;
+                logger.warn("connect failed by " + ExceptionUtils.getStackTrace(e));
+            }
+
+        }
+
+        public MysqlConnection getMysqlConnection() {
+            return mysqlConnection;
+        }
+    }
+
+    // 处理主备切换的逻辑
+    public void doSwitch() {
+        AuthenticationInfo newRunningInfo = (runningInfo.equals(masterInfo) ? standbyInfo : masterInfo);
+        this.doSwitch(newRunningInfo);
+    }
+
+    public void doSwitch(AuthenticationInfo newRunningInfo) {
+        // 1. 需要停止当前正在复制的过程
+        // 2. 找到新的position点
+        // 3. 重新建立链接，开始复制数据
+        // 切换ip
+        String alarmMessage = null;
+
+        if (this.runningInfo.equals(newRunningInfo)) {
+            alarmMessage = "same runingInfo switch again : " + runningInfo.getAddress().toString();
+            logger.warn(alarmMessage);
+            return;
+        }
+
+        if (newRunningInfo == null) {
+            alarmMessage = "no standby config, just do nothing, will continue try:"
+                           + runningInfo.getAddress().toString();
+            logger.warn(alarmMessage);
+            sendAlarm(destination, alarmMessage);
+            return;
+        } else {
+            stop();
+            alarmMessage = "try to ha switch, old:" + runningInfo.getAddress().toString() + ", new:"
+                           + newRunningInfo.getAddress().toString();
+            logger.warn(alarmMessage);
+            sendAlarm(destination, alarmMessage);
+            runningInfo = newRunningInfo;
+            start();
+        }
+    }
+
+    // =================== helper method =================
+
+    private MysqlConnection buildMysqlConnection(AuthenticationInfo runningInfo) {
+        MysqlConnection connection = new MysqlConnection(runningInfo.getAddress(),
+            runningInfo.getUsername(),
+            runningInfo.getPassword(),
+            connectionCharsetNumber,
+            runningInfo.getDefaultDatabaseName());
+        connection.getConnector().setReceiveBufferSize(receiveBufferSize);
+        connection.getConnector().setSendBufferSize(sendBufferSize);
+        connection.getConnector().setSoTimeout(defaultConnectionTimeoutInSeconds * 1000);
+        connection.setCharset(connectionCharset);
+        connection.setSlaveId(this.slaveId);
+        return connection;
+    }
+
+    protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
+        EntryPosition startPosition = findStartPositionInternal(connection);
+        if (needTransactionPosition.get()) {
+            logger.warn("prepare to find last position : {}", startPosition.toString());
+            Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
+            if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
+                logger.warn("find new start Transaction Position , old : {} , new : {}",
+                    startPosition.getPosition(),
+                    preTransactionStartPosition);
+                startPosition.setPosition(preTransactionStartPosition);
+            }
+            needTransactionPosition.compareAndSet(true, false);
+        }
+        return startPosition;
+    }
+
+    protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
+        MysqlConnection mysqlConnection = (MysqlConnection) connection;
+        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+        if (logPosition == null) {// 找不到历史成功记录
+            EntryPosition entryPosition = null;
+            if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
+                entryPosition = masterPosition;
+            } else if (standbyInfo != null
+                       && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
+                entryPosition = standbyPosition;
+            }
+
+            if (entryPosition == null) {
+                entryPosition = findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
+            }
+
+            // 判断一下是否需要按时间订阅
+            if (StringUtils.isEmpty(entryPosition.getJournalName())) {
+                // 如果没有指定binlogName，尝试按照timestamp进行查找
+                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
+                    logger.warn("prepare to find start position {}:{}:{}",
+                        new Object[] { "", "", entryPosition.getTimestamp() });
+                    return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
+                } else {
+                    logger.warn("prepare to find start position just show master status");
+                    return findEndPosition(mysqlConnection); // 默认从当前最后一个位置进行消费
+                }
+            } else {
+                if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
+                    // 如果指定binlogName + offest，直接返回
+                    logger.warn("prepare to find start position {}:{}:{}",
+                        new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(), "" });
+                    return entryPosition;
+                } else {
+                    EntryPosition specificLogFilePosition = null;
+                    if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
+                        // 如果指定binlogName +
+                        // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
+                        EntryPosition endPosition = findEndPosition(mysqlConnection);
+                        if (endPosition != null) {
+                            logger.warn("prepare to find start position {}:{}:{}",
+                                new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
+                            specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
+                                entryPosition.getTimestamp(),
+                                endPosition,
+                                entryPosition.getJournalName());
+                        }
+                    }
+
+                    if (specificLogFilePosition == null) {
+                        // position不存在，从文件头开始
+                        entryPosition.setPosition(BINLOG_START_OFFEST);
+                        return entryPosition;
+                    } else {
+                        return specificLogFilePosition;
+                    }
+                }
+            }
+        } else {
+            if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
+                logger.warn("prepare to find start position just last position");
+                return logPosition.getPostion();
+            } else {
+                // 针对切换的情况，考虑回退时间
+                long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
+                logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "",
+                        logPosition.getPostion().getTimestamp() });
+                return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
+            }
+        }
+    }
+
+    // 根据想要的position，可能这个position对应的记录为rowdata，需要找到事务头，避免丢数据
+    // 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
+    private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition)
+                                                                                                                 throws IOException {
+        // 尝试找到一个合适的位置
+        final AtomicBoolean reDump = new AtomicBoolean(false);
+        mysqlConnection.reconnect();
+        mysqlConnection.seek(entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction() {
+
+            private LogPosition lastPosition;
+
+            public boolean sink(LogEvent event) {
+                try {
+                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+                    if (entry == null) {
+                        return true;
+                    }
+
+                    // 直接查询第一条业务数据，确认是否为事务Begin/End
+                    if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType()
+                        || CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) {
+                        lastPosition = buildLastPosition(entry);
+                        return false;
+                    } else {
+                        reDump.set(true);
+                        lastPosition = buildLastPosition(entry);
+                        return false;
+                    }
+                } catch (Exception e) {
+                    // 上一次记录的poistion可能为一条update/insert/delete变更事件，直接进行dump的话，会缺少tableMap事件，导致tableId未进行解析
+                    processError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
+                    reDump.set(true);
+                    return false;
+                }
+            }
+        });
+        // 针对开始的第一条为非Begin记录，需要从该binlog扫描
+        if (reDump.get()) {
+            final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
+            mysqlConnection.reconnect();
+            mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction() {
+
+                private LogPosition lastPosition;
+
+                public boolean sink(LogEvent event) {
+                    try {
+                        CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+                        if (entry == null) {
+                            return true;
+                        }
+
+                        // 直接查询第一条业务数据，确认是否为事务Begin
+                        // 记录一下transaction begin position
+                        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                            && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
+                            preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
+                        }
+
+                        if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
+                            return false;// 退出
+                        }
+
+                        lastPosition = buildLastPosition(entry);
+                    } catch (Exception e) {
+                        processError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
+                        return false;
+                    }
+
+                    return running;
+                }
+            });
+
+            // 判断一下找到的最接近position的事务头的位置
+            if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
+                logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
+                throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
+            }
+            return preTransactionStartPosition.get();
+        } else {
+            return entryPosition.getPosition();
+        }
+    }
+
+    // 根据时间查找binlog位置
+    private EntryPosition findByStartTimeStamp(MysqlConnection mysqlConnection, Long startTimestamp) {
+        EntryPosition endPosition = findEndPosition(mysqlConnection);
+        EntryPosition startPosition = findStartPosition(mysqlConnection);
+        String maxBinlogFileName = endPosition.getJournalName();
+        String minBinlogFileName = startPosition.getJournalName();
+        logger.info("show master status to set search end condition:{} ", endPosition);
+        String startSearchBinlogFile = endPosition.getJournalName();
+        boolean shouldBreak = false;
+        while (running && !shouldBreak) {
+            try {
+                EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
+                    startTimestamp,
+                    endPosition,
+                    startSearchBinlogFile);
+                if (entryPosition == null) {
+                    if (StringUtils.equalsIgnoreCase(minBinlogFileName, startSearchBinlogFile)) {
+                        // 已经找到最早的一个binlog，没必要往前找了
+                        shouldBreak = true;
+                        logger.warn("Didn't find the corresponding binlog files from {} to {}",
+                            minBinlogFileName,
+                            maxBinlogFileName);
+                    } else {
+                        // 继续往前找
+                        int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1));
+                        if (binlogSeqNum <= 1) {
+                            logger.warn("Didn't find the corresponding binlog files");
+                            shouldBreak = true;
+                        } else {
+                            int nextBinlogSeqNum = binlogSeqNum - 1;
+                            String binlogFileNamePrefix = startSearchBinlogFile.substring(0,
+                                startSearchBinlogFile.indexOf(".") + 1);
+                            String binlogFileNameSuffix = String.format("%06d", nextBinlogSeqNum);
+                            startSearchBinlogFile = binlogFileNamePrefix + binlogFileNameSuffix;
+                        }
+                    }
+                } else {
+                    logger.info("found and return:{} in findByStartTimeStamp operation.", entryPosition);
+                    return entryPosition;
+                }
+            } catch (Exception e) {
+                logger.warn("the binlogfile:{} doesn't exist, to continue to search the next binlogfile , caused by {}",
+                    startSearchBinlogFile,
+                    ExceptionUtils.getFullStackTrace(e));
+                int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1));
+                if (binlogSeqNum <= 1) {
+                    logger.warn("Didn't find the corresponding binlog files");
+                    shouldBreak = true;
+                } else {
+                    int nextBinlogSeqNum = binlogSeqNum - 1;
+                    String binlogFileNamePrefix = startSearchBinlogFile.substring(0,
+                        startSearchBinlogFile.indexOf(".") + 1);
+                    String binlogFileNameSuffix = String.format("%06d", nextBinlogSeqNum);
+                    startSearchBinlogFile = binlogFileNamePrefix + binlogFileNameSuffix;
+                }
+            }
+        }
+        // 找不到
+        return null;
+    }
+
+    /**
+     * 查询当前的binlog位置
+     */
+    private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
+        try {
+            ResultSetPacket packet = mysqlConnection.query("show master status");
+            List fields = packet.getFieldValues();
+            if (CollectionUtils.isEmpty(fields)) {
+                throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
+            }
+            EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
+            return endPosition;
+        } catch (IOException e) {
+            throw new CanalParseException("command : 'show master status' has an error!", e);
+        }
+    }
+
+    /**
+     * 查询当前的binlog位置
+     */
+    private EntryPosition findStartPosition(MysqlConnection mysqlConnection) {
+        try {
+            ResultSetPacket packet = mysqlConnection.query("show binlog events limit 1");
+            List fields = packet.getFieldValues();
+            if (CollectionUtils.isEmpty(fields)) {
+                throw new CanalParseException("command : 'show binlog events limit 1' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
+            }
+            EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
+            return endPosition;
+        } catch (IOException e) {
+            throw new CanalParseException("command : 'show binlog events limit 1' has an error!", e);
+        }
+
+    }
+
+    /**
+     * 查询当前的slave视图的binlog位置
+     */
+    @SuppressWarnings("unused")
+    private SlaveEntryPosition findSlavePosition(MysqlConnection mysqlConnection) {
+        try {
+            ResultSetPacket packet = mysqlConnection.query("show slave status");
+            List names = packet.getFieldDescriptors();
+            List fields = packet.getFieldValues();
+            if (CollectionUtils.isEmpty(fields)) {
+                return null;
+            }
+
+            int i = 0;
+            Map maps = new HashMap(names.size(), 1f);
+            for (FieldPacket name : names) {
+                maps.put(name.getName(), fields.get(i));
+                i++;
+            }
+
+            String errno = maps.get("Last_Errno");
+            String slaveIORunning = maps.get("Slave_IO_Running"); // Slave_SQL_Running
+            String slaveSQLRunning = maps.get("Slave_SQL_Running"); // Slave_SQL_Running
+            if ((!"0".equals(errno)) || (!"Yes".equalsIgnoreCase(slaveIORunning))
+                || (!"Yes".equalsIgnoreCase(slaveSQLRunning))) {
+                logger.warn("Ignoring failed slave: " + mysqlConnection.getConnector().getAddress() + ", Last_Errno = "
+                            + errno + ", Slave_IO_Running = " + slaveIORunning + ", Slave_SQL_Running = "
+                            + slaveSQLRunning);
+                return null;
+            }
+
+            String masterHost = maps.get("Master_Host");
+            String masterPort = maps.get("Master_Port");
+            String binlog = maps.get("Master_Log_File");
+            String position = maps.get("Exec_Master_Log_Pos");
+            return new SlaveEntryPosition(binlog, Long.valueOf(position), masterHost, masterPort);
+        } catch (IOException e) {
+            logger.error("find slave position error", e);
+        }
+
+        return null;
+    }
+
+    /**
+     * 根据给定的时间戳，在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。
+     * 针对最后一个binlog会给定endPosition，避免无尽的查询
+     */
+    private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection,
+                                                              final Long startTimestamp,
+                                                              final EntryPosition endPosition,
+                                                              final String searchBinlogFile) {
+
+        final LogPosition logPosition = new LogPosition();
+        try {
+            mysqlConnection.reconnect();
+            // 开始遍历文件
+            mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction() {
+
+                private LogPosition lastPosition;
+
+                public boolean sink(LogEvent event) {
+                    EntryPosition entryPosition = null;
+                    try {
+                        CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
+                        if (entry == null) {
+                            return true;
+                        }
+
+                        String logfilename = entry.getHeader().getLogfileName();
+                        Long logfileoffset = entry.getHeader().getLogfileOffset();
+                        Long logposTimestamp = entry.getHeader().getExecuteTime();
+
+                        if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
+                            logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] {
+                                    logfilename, logfileoffset, logposTimestamp, startTimestamp });
+                            // 寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
+                            if (logposTimestamp >= startTimestamp) {
+                                return false;
+                            }
+                        }
+
+                        if (StringUtils.equals(endPosition.getJournalName(), logfilename)
+                            && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
+                            return false;
+                        }
+
+                        // 记录一下上一个事务结束的位置，即下一个事务的position
+                        // position = current +
+                        // data.length，代表该事务的下一条offest，避免多余的事务重复
+                        if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
+                            entryPosition = new EntryPosition(logfilename,
+                                logfileoffset + event.getEventLen(),
+                                logposTimestamp);
+                            logger.debug("set {} to be pending start position before finding another proper one...",
+                                entryPosition);
+                            logPosition.setPostion(entryPosition);
+                        }
+
+                        lastPosition = buildLastPosition(entry);
+                    } catch (Exception e) {
+                        processError(e, lastPosition, searchBinlogFile, 4L);
+                    }
+
+                    return running;
+                }
+            });
+
+        } catch (IOException e) {
+            logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
+        }
+
+        if (logPosition.getPostion() != null) {
+            return logPosition.getPostion();
+        } else {
+            return null;
+        }
+    }
+
+    protected Entry parseAndProfilingIfNecessary(LogEvent bod) throws Exception {
+        long startTs = -1;
+        boolean enabled = getProfilingEnabled();
+        if (enabled) {
+            startTs = System.currentTimeMillis();
+        }
+        CanalEntry.Entry event = binlogParser.parse(bod);
+        if (enabled) {
+            this.parsingInterval = System.currentTimeMillis() - startTs;
+        }
+
+        if (parsedEventCount.incrementAndGet() < 0) {
+            parsedEventCount.set(0);
+        }
+        return event;
+    }
+
+    // ===================== setter / getter ========================
+
+    public void setDefaultConnectionTimeoutInSeconds(int defaultConnectionTimeoutInSeconds) {
+        this.defaultConnectionTimeoutInSeconds = defaultConnectionTimeoutInSeconds;
+    }
+
+    public void setReceiveBufferSize(int receiveBufferSize) {
+        this.receiveBufferSize = receiveBufferSize;
+    }
+
+    public void setSendBufferSize(int sendBufferSize) {
+        this.sendBufferSize = sendBufferSize;
+    }
+
+    public void setMasterInfo(AuthenticationInfo masterInfo) {
+        this.masterInfo = masterInfo;
+    }
+
+    public void setStandbyInfo(AuthenticationInfo standbyInfo) {
+        this.standbyInfo = standbyInfo;
+    }
+
+    public void setMasterPosition(EntryPosition masterPosition) {
+        this.masterPosition = masterPosition;
+    }
+
+    public void setStandbyPosition(EntryPosition standbyPosition) {
+        this.standbyPosition = standbyPosition;
+    }
+
+    public void setSlaveId(long slaveId) {
+        this.slaveId = slaveId;
+    }
+
+    public void setDetectingSQL(String detectingSQL) {
+        this.detectingSQL = detectingSQL;
+    }
+
+    public void setDetectingIntervalInSeconds(Integer detectingIntervalInSeconds) {
+        this.detectingIntervalInSeconds = detectingIntervalInSeconds;
+    }
+
+    public void setDetectingEnable(boolean detectingEnable) {
+        this.detectingEnable = detectingEnable;
+    }
+
+    public void setFallbackIntervalInSeconds(int fallbackIntervalInSeconds) {
+        this.fallbackIntervalInSeconds = fallbackIntervalInSeconds;
+    }
+
+    public CanalHAController getHaController() {
+        return haController;
+    }
+
+    public void setHaController(CanalHAController haController) {
+        this.haController = haController;
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/SlaveEntryPosition.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/SlaveEntryPosition.java
new file mode 100644
index 00000000..67dde4e2
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/SlaveEntryPosition.java
@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+
+/**
+ * slave status状态的信息
+ * 
+ * @author jianghang 2013-1-23 下午09:42:18
+ * @version 1.0.0
+ */
+public class SlaveEntryPosition extends EntryPosition {
+
+    private static final long serialVersionUID = 5271424551446372093L;
+    private final String      masterHost;
+    private final String      masterPort;
+
+    public SlaveEntryPosition(String fileName, long position, String masterHost, String masterPort){
+        super(fileName, position);
+
+        this.masterHost = masterHost;
+        this.masterPort = masterPort;
+    }
+
+    public String getMasterHost() {
+        return masterHost;
+    }
+
+    public String getMasterPort() {
+        return masterPort;
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java
new file mode 100644
index 00000000..0742e391
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/DirectLogFetcher.java
@@ -0,0 +1,180 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.SocketChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.taobao.tddl.dbsync.binlog.LogFetcher;
+
+/**
+ * 基于socket的logEvent实现
+ * 
+ * @author jianghang 2013-1-14 下午07:39:30
+ * @version 1.0.0
+ */
+public class DirectLogFetcher extends LogFetcher {
+
+    protected static final Logger logger            = LoggerFactory.getLogger(DirectLogFetcher.class);
+
+    /** Command to dump binlog */
+    public static final byte      COM_BINLOG_DUMP   = 18;
+
+    /** Packet header sizes */
+    public static final int       NET_HEADER_SIZE   = 4;
+    public static final int       SQLSTATE_LENGTH   = 5;
+
+    /** Packet offsets */
+    public static final int       PACKET_LEN_OFFSET = 0;
+    public static final int       PACKET_SEQ_OFFSET = 3;
+
+    /** Maximum packet length */
+    public static final int       MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
+
+    private SocketChannel         channel;
+
+    // private BufferedInputStream input;
+
+    public DirectLogFetcher(){
+        super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public DirectLogFetcher(final int initialCapacity){
+        super(initialCapacity, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public DirectLogFetcher(final int initialCapacity, final float growthFactor){
+        super(initialCapacity, growthFactor);
+    }
+
+    public void start(SocketChannel channel) throws IOException {
+        this.channel = channel;
+        // 和mysql driver一样，提供buffer机制，提升读取binlog速度
+        // this.input = new
+        // BufferedInputStream(channel.socket().getInputStream(), 16384);
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
+     */
+    public boolean fetch() throws IOException {
+        try {
+            // Fetching packet header from input.
+            if (!fetch0(0, NET_HEADER_SIZE)) {
+                logger.warn("Reached end of input stream while fetching header");
+                return false;
+            }
+
+            // Fetching the first packet(may a multi-packet).
+            int netlen = getUint24(PACKET_LEN_OFFSET);
+            int netnum = getUint8(PACKET_SEQ_OFFSET);
+            if (!fetch0(NET_HEADER_SIZE, netlen)) {
+                logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
+                return false;
+            }
+
+            // Detecting error code.
+            final int mark = getUint8(NET_HEADER_SIZE);
+            if (mark != 0) {
+                if (mark == 255) // error from master
+                {
+                    // Indicates an error, for example trying to fetch from
+                    // wrong
+                    // binlog position.
+                    position = NET_HEADER_SIZE + 1;
+                    final int errno = getInt16();
+                    String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
+                    String errmsg = getFixString(limit - position);
+                    throw new IOException("Received error packet:" + " errno = " + errno + ", sqlstate = " + sqlstate
+                                          + " errmsg = " + errmsg);
+                } else if (mark == 254) {
+                    // Indicates end of stream. It's not clear when this would
+                    // be sent.
+                    logger.warn("Received EOF packet from server, apparent"
+                                + " master disconnected. It's may be duplicate slaveId , check instance config");
+                    return false;
+                } else {
+                    // Should not happen.
+                    throw new IOException("Unexpected response " + mark + " while fetching binlog: packet #" + netnum
+                                          + ", len = " + netlen);
+                }
+            }
+
+            // The first packet is a multi-packet, concatenate the packets.
+            while (netlen == MAX_PACKET_LENGTH) {
+                if (!fetch0(0, NET_HEADER_SIZE)) {
+                    logger.warn("Reached end of input stream while fetching header");
+                    return false;
+                }
+
+                netlen = getUint24(PACKET_LEN_OFFSET);
+                netnum = getUint8(PACKET_SEQ_OFFSET);
+                if (!fetch0(limit, netlen)) {
+                    logger.warn("Reached end of input stream: packet #" + netnum + ", len = " + netlen);
+                    return false;
+                }
+            }
+
+            // Preparing buffer variables to decoding.
+            origin = NET_HEADER_SIZE + 1;
+            position = origin;
+            limit -= origin;
+            return true;
+        } catch (SocketTimeoutException e) {
+            close(); /* Do cleanup */
+            logger.error("Socket timeout expired, closing connection", e);
+            throw e;
+        } catch (InterruptedIOException e) {
+            close(); /* Do cleanup */
+            logger.info("I/O interrupted while reading from client socket", e);
+            throw e;
+        } catch (ClosedByInterruptException e) {
+            close(); /* Do cleanup */
+            logger.info("I/O interrupted while reading from client socket", e);
+            throw e;
+        } catch (IOException e) {
+            close(); /* Do cleanup */
+            logger.error("I/O error while reading from client socket", e);
+            throw e;
+        }
+    }
+
+    private final boolean fetch0(final int off, final int len) throws IOException {
+        ensureCapacity(off + len);
+
+        ByteBuffer buffer = ByteBuffer.wrap(this.buffer, off, len);
+        while (buffer.hasRemaining()) {
+            int readNum = channel.read(buffer);
+            if (readNum == -1) {
+                throw new IOException("Unexpected End Stream");
+            }
+        }
+
+        // for (int count, n = 0; n < len; n += count) {
+        // if (0 > (count = input.read(buffer, off + n, len - n))) {
+        // // Reached end of input stream
+        // return false;
+        // }
+        // }
+
+        if (limit < off + len) limit = off + len;
+        return true;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
+     */
+    public void close() throws IOException {
+        // do nothing
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
new file mode 100644
index 00000000..14fd3498
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
@@ -0,0 +1,707 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.sql.Types;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.exception.TableIdNotFoundException;
+import com.alibaba.otter.canal.parse.inbound.BinlogParser;
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.protocol.CanalEntry.Pair;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin;
+import com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd;
+import com.alibaba.otter.canal.protocol.CanalEntry.Type;
+import com.google.protobuf.ByteString;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+
+/**
+ * 基于{@linkplain LogEvent}转化为Entry对象的处理
+ * 
+ * @author jianghang 2013-1-17 下午02:41:14
+ * @version 1.0.0
+ */
+public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogParser {
+
+    public static final String          ISO_8859_1          = "ISO-8859-1";
+    public static final String          UTF_8               = "UTF-8";
+    public static final int             TINYINT_MAX_VALUE   = 256;
+    public static final int             SMALLINT_MAX_VALUE  = 65536;
+    public static final int             MEDIUMINT_MAX_VALUE = 16777216;
+    public static final long            INTEGER_MAX_VALUE   = 4294967296L;
+    public static final BigInteger      BIGINT_MAX_VALUE    = new BigInteger("18446744073709551616");
+    public static final int             version             = 1;
+    public static final String          BEGIN               = "BEGIN";
+    public static final String          COMMIT              = "COMMIT";
+    public static final Logger          logger              = LoggerFactory.getLogger(LogEventConvert.class);
+
+    private volatile AviaterRegexFilter nameFilter;                                                          // 运行时引用可能会有变化，比如规则发生变化时
+    private volatile AviaterRegexFilter nameBlackFilter;
+
+    private TableMetaCache              tableMetaCache;
+    private String                      binlogFileName      = "mysql-bin.000001";
+    private Charset                     charset             = Charset.defaultCharset();
+    private boolean                     filterQueryDcl      = false;
+    private boolean                     filterQueryDml      = false;
+    private boolean                     filterQueryDdl      = false;
+    // 是否跳过table相关的解析异常,比如表不存在或者列数量不匹配,issue 92
+    private boolean                     filterTableError    = false;
+
+    public Entry parse(LogEvent logEvent) throws CanalParseException {
+        if (logEvent == null || logEvent instanceof UnknownLogEvent) {
+            return null;
+        }
+
+        int eventType = logEvent.getHeader().getType();
+        switch (eventType) {
+            case LogEvent.ROTATE_EVENT:
+                binlogFileName = ((RotateLogEvent) logEvent).getFilename();
+                break;
+            case LogEvent.QUERY_EVENT:
+                return parseQueryEvent((QueryLogEvent) logEvent);
+            case LogEvent.XID_EVENT:
+                return parseXidEvent((XidLogEvent) logEvent);
+            case LogEvent.TABLE_MAP_EVENT:
+                break;
+            case LogEvent.WRITE_ROWS_EVENT_V1:
+            case LogEvent.WRITE_ROWS_EVENT:
+                return parseRowsEvent((WriteRowsLogEvent) logEvent);
+            case LogEvent.UPDATE_ROWS_EVENT_V1:
+            case LogEvent.UPDATE_ROWS_EVENT:
+                return parseRowsEvent((UpdateRowsLogEvent) logEvent);
+            case LogEvent.DELETE_ROWS_EVENT_V1:
+            case LogEvent.DELETE_ROWS_EVENT:
+                return parseRowsEvent((DeleteRowsLogEvent) logEvent);
+            case LogEvent.ROWS_QUERY_LOG_EVENT:
+                return parseRowsQueryEvent((RowsQueryLogEvent) logEvent);
+            case LogEvent.ANNOTATE_ROWS_EVENT:
+                return parseAnnotateRowsEvent((AnnotateRowsEvent) logEvent);
+            case LogEvent.USER_VAR_EVENT:
+                return parseUserVarLogEvent((UserVarLogEvent) logEvent);
+            case LogEvent.INTVAR_EVENT:
+                return parseIntrvarLogEvent((IntvarLogEvent) logEvent);
+            case LogEvent.RAND_EVENT:
+                return parseRandLogEvent((RandLogEvent) logEvent);
+            default:
+                break;
+        }
+
+        return null;
+    }
+
+    public void reset() {
+        // do nothing
+        binlogFileName = "mysql-bin.000001";
+        if (tableMetaCache != null) {
+            tableMetaCache.clearTableMeta();
+        }
+    }
+
+    private Entry parseQueryEvent(QueryLogEvent event) {
+        String queryString = event.getQuery();
+        if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
+            TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
+            Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
+            return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
+        } else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
+            TransactionEnd transactionEnd = createTransactionEnd(0L); // MyISAM可能不会有xid事件
+            Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
+            return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
+        } else {
+            // DDL语句处理
+            DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
+
+            String schemaName = event.getDbName();
+            if (StringUtils.isNotEmpty(result.getSchemaName())) {
+                schemaName = result.getSchemaName();
+            }
+
+            String tableName = result.getTableName();
+            EventType type = EventType.QUERY;
+            // fixed issue https://github.com/alibaba/canal/issues/58
+            if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE
+                || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE
+                || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX
+                || result.getType() == EventType.DINDEX) { // 针对DDL类型
+
+                if (filterQueryDdl) {
+                    return null;
+                }
+
+                type = result.getType();
+                if (StringUtils.isEmpty(tableName)
+                    || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
+                    // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
+                    throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: "
+                                                  + queryString + " , and DdlResult: " + result.toString());
+                    // return null;
+                } else {
+                    // check name filter
+                    String name = schemaName + "." + tableName;
+                    if (nameFilter != null && !nameFilter.filter(name)) {
+                        if (result.getType() == EventType.RENAME) {
+                            // rename校验只要源和目标满足一个就进行操作
+                            if (nameFilter != null
+                                && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
+                                return null;
+                            }
+                        } else {
+                            // 其他情况返回null
+                            return null;
+                        }
+                    }
+
+                    if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
+                        if (result.getType() == EventType.RENAME) {
+                            // rename校验只要源和目标满足一个就进行操作
+                            if (nameBlackFilter != null
+                                && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
+                                return null;
+                            }
+                        } else {
+                            // 其他情况返回null
+                            return null;
+                        }
+                    }
+                }
+            } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE
+                       || result.getType() == EventType.DELETE) {
+                // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
+                if (filterQueryDml) {
+                    return null;
+                }
+            } else if (filterQueryDcl) {
+                return null;
+            }
+
+            // 更新下table meta cache
+            if (tableMetaCache != null
+                && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
+                if (StringUtils.isNotEmpty(tableName)) {
+                    // 如果解析到了正确的表信息，则根据全名进行清除
+                    tableMetaCache.clearTableMeta(schemaName, tableName);
+                } else {
+                    // 如果无法解析正确的表信息，则根据schema进行清除
+                    tableMetaCache.clearTableMetaWithSchemaName(schemaName);
+                }
+            }
+
+            Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
+            RowChange.Builder rowChangeBuider = RowChange.newBuilder();
+            if (result.getType() != EventType.QUERY) {
+                rowChangeBuider.setIsDdl(true);
+            }
+            rowChangeBuider.setSql(queryString);
+            if (StringUtils.isNotEmpty(event.getDbName())) {// 可能为空
+                rowChangeBuider.setDdlSchemaName(event.getDbName());
+            }
+            rowChangeBuider.setEventType(result.getType());
+            return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
+        }
+    }
+
+    private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+        // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
+        String queryString = null;
+        try {
+            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
+            return buildQueryEntry(queryString, event.getHeader());
+        } catch (UnsupportedEncodingException e) {
+            throw new CanalParseException(e);
+        }
+    }
+
+    private Entry parseAnnotateRowsEvent(AnnotateRowsEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+        // mariaDb支持，需要设置binlog_annotate_row_events=true，可详细打印原始DML语句
+        String queryString = null;
+        try {
+            queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
+            return buildQueryEntry(queryString, event.getHeader());
+        } catch (UnsupportedEncodingException e) {
+            throw new CanalParseException(e);
+        }
+    }
+
+    private Entry parseUserVarLogEvent(UserVarLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+
+        return buildQueryEntry(event.getQuery(), event.getHeader());
+    }
+
+    private Entry parseIntrvarLogEvent(IntvarLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+
+        return buildQueryEntry(event.getQuery(), event.getHeader());
+    }
+
+    private Entry parseRandLogEvent(RandLogEvent event) {
+        if (filterQueryDml) {
+            return null;
+        }
+
+        return buildQueryEntry(event.getQuery(), event.getHeader());
+    }
+
+    private Entry parseXidEvent(XidLogEvent event) {
+        TransactionEnd transactionEnd = createTransactionEnd(event.getXid());
+        Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
+        return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
+    }
+
+    private Entry parseRowsEvent(RowsLogEvent event) {
+        try {
+            TableMapLogEvent table = event.getTable();
+            if (table == null) {
+                // tableId对应的记录不存在
+                throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
+            }
+
+            String fullname = table.getDbName() + "." + table.getTableName();
+            // check name filter
+            if (nameFilter != null && !nameFilter.filter(fullname)) {
+                return null;
+            }
+            if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
+                return null;
+            }
+
+            EventType eventType = null;
+            int type = event.getHeader().getType();
+            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
+                eventType = EventType.INSERT;
+            } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
+                eventType = EventType.UPDATE;
+            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
+                eventType = EventType.DELETE;
+            } else {
+                throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
+            }
+
+            Header header = createHeader(binlogFileName,
+                event.getHeader(),
+                table.getDbName(),
+                table.getTableName(),
+                eventType);
+            RowChange.Builder rowChangeBuider = RowChange.newBuilder();
+            rowChangeBuider.setTableId(event.getTableId());
+            rowChangeBuider.setIsDdl(false);
+
+            rowChangeBuider.setEventType(eventType);
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+            BitSet columns = event.getColumns();
+            BitSet changeColumns = event.getColumns();
+            boolean tableError = false;
+            TableMeta tableMeta = null;
+            if (tableMetaCache != null) {// 入错存在table meta cache
+                tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
+                if (tableMeta == null) {
+                    tableError = true;
+                    if (!filterTableError) {
+                        throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
+                    }
+                }
+            }
+
+            while (buffer.nextOneRow(columns)) {
+                // 处理row记录
+                RowData.Builder rowDataBuilder = RowData.newBuilder();
+                if (EventType.INSERT == eventType) {
+                    // insert的记录放在before字段中
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
+                } else if (EventType.DELETE == eventType) {
+                    // delete的记录放在before字段中
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
+                } else {
+                    // update需要处理before/after
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
+                    if (!buffer.nextOneRow(changeColumns)) {
+                        rowChangeBuider.addRowDatas(rowDataBuilder.build());
+                        break;
+                    }
+
+                    tableError |= parseOneRow(rowDataBuilder, event, buffer, event.getChangeColumns(), true, tableMeta);
+                }
+
+                rowChangeBuider.addRowDatas(rowDataBuilder.build());
+            }
+
+            RowChange rowChange = rowChangeBuider.build();
+            if (tableError) {
+                Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
+                logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
+                return null;
+            } else {
+                Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
+                return entry;
+            }
+        } catch (Exception e) {
+            throw new CanalParseException("parse row data failed.", e);
+        }
+    }
+
+    private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols,
+                                boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
+        final int columnCnt = event.getTable().getColumnCnt();
+        final ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
+
+        boolean tableError = false;
+        // check table fileds count，只能处理加字段
+        if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
+            // online ddl增加字段操作步骤：
+            // 1. 新增一张临时表，将需要做ddl表的数据全量导入
+            // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
+            // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作
+            // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作
+            // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作
+            tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false);// 强制重新获取一次
+            if (tableMeta == null) {
+                tableError = true;
+                if (!filterTableError) {
+                    throw new CanalParseException("not found [" + event.getTable().getDbName() + "."
+                                                  + event.getTable().getTableName() + "] in db , pls check!");
+                }
+            }
+
+            // 在做一次判断
+            if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
+                tableError = true;
+                if (!filterTableError) {
+                    throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + ","
+                                                  + columnInfo.length + " vs " + tableMeta.getFileds().size());
+                }
+            }
+        }
+
+        for (int i = 0; i < columnCnt; i++) {
+            ColumnInfo info = columnInfo[i];
+            Column.Builder columnBuilder = Column.newBuilder();
+
+            FieldMeta fieldMeta = null;
+            if (tableMeta != null && !tableError) {
+                // 处理file meta
+                fieldMeta = tableMeta.getFileds().get(i);
+                columnBuilder.setName(fieldMeta.getColumnName());
+                columnBuilder.setIsKey(fieldMeta.isKey());
+                // 增加mysql type类型,issue 73
+                columnBuilder.setMysqlType(fieldMeta.getColumnType());
+            }
+            columnBuilder.setIndex(i);
+            columnBuilder.setIsNull(false);
+
+            // fixed issue
+            // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
+            boolean isBinary = false;
+            if (fieldMeta != null) {
+                if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
+                    isBinary = true;
+                } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
+                    isBinary = true;
+                }
+            }
+            buffer.nextValue(info.type, info.meta, isBinary);
+
+            int javaType = buffer.getJavaType();
+            if (buffer.isNull()) {
+                columnBuilder.setIsNull(true);
+            } else {
+                final Serializable value = buffer.getValue();
+                // 处理各种类型
+                switch (javaType) {
+                    case Types.INTEGER:
+                    case Types.TINYINT:
+                    case Types.SMALLINT:
+                    case Types.BIGINT:
+                        // 处理unsigned类型
+                        Number number = (Number) value;
+                        if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
+                            switch (buffer.getLength()) {
+                                case 1: /* MYSQL_TYPE_TINY */
+                                    columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE
+                                                                                          + number.intValue())));
+                                    javaType = Types.SMALLINT; // 往上加一个量级
+                                    break;
+
+                                case 2: /* MYSQL_TYPE_SHORT */
+                                    columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE
+                                                                                          + number.intValue())));
+                                    javaType = Types.INTEGER; // 往上加一个量级
+                                    break;
+
+                                case 3: /* MYSQL_TYPE_INT24 */
+                                    columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE
+                                                                                          + number.intValue())));
+                                    javaType = Types.INTEGER; // 往上加一个量级
+                                    break;
+
+                                case 4: /* MYSQL_TYPE_LONG */
+                                    columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE
+                                                                                       + number.longValue())));
+                                    javaType = Types.BIGINT; // 往上加一个量级
+                                    break;
+
+                                case 8: /* MYSQL_TYPE_LONGLONG */
+                                    columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue()))
+                                        .toString());
+                                    javaType = Types.DECIMAL; // 往上加一个量级，避免执行出错
+                                    break;
+                            }
+                        } else {
+                            // 对象为number类型，直接valueof即可
+                            columnBuilder.setValue(String.valueOf(value));
+                        }
+                        break;
+                    case Types.REAL: // float
+                    case Types.DOUBLE: // double
+                        // 对象为number类型，直接valueof即可
+                        columnBuilder.setValue(String.valueOf(value));
+                        break;
+                    case Types.BIT:// bit
+                        // 对象为number类型
+                        columnBuilder.setValue(String.valueOf(value));
+                        break;
+                    case Types.DECIMAL:
+                        columnBuilder.setValue(((BigDecimal) value).toPlainString());
+                        break;
+                    case Types.TIMESTAMP:
+                        // 修复时间边界值
+                        // String v = value.toString();
+                        // v = v.substring(0, v.length() - 2);
+                        // columnBuilder.setValue(v);
+                        // break;
+                    case Types.TIME:
+                    case Types.DATE:
+                        // 需要处理year
+                        columnBuilder.setValue(value.toString());
+                        break;
+                    case Types.BINARY:
+                    case Types.VARBINARY:
+                    case Types.LONGVARBINARY:
+                        // fixed text encoding
+                        // https://github.com/AlibabaTech/canal/issues/18
+                        // mysql binlog中blob/text都处理为blob类型，需要反查table
+                        // meta，按编码解析text
+                        if (fieldMeta != null && isText(fieldMeta.getColumnType())) {
+                            columnBuilder.setValue(new String((byte[]) value, charset));
+                            javaType = Types.CLOB;
+                        } else {
+                            // byte数组，直接使用iso-8859-1保留对应编码，浪费内存
+                            columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
+                            javaType = Types.BLOB;
+                        }
+                        break;
+                    case Types.CHAR:
+                    case Types.VARCHAR:
+                        columnBuilder.setValue(value.toString());
+                        break;
+                    default:
+                        columnBuilder.setValue(value.toString());
+                }
+
+            }
+
+            columnBuilder.setSqlType(javaType);
+            // 设置是否update的标记位
+            columnBuilder.setUpdated(isAfter
+                                     && isUpdate(rowDataBuilder.getBeforeColumnsList(),
+                                         columnBuilder.getIsNull() ? null : columnBuilder.getValue(),
+                                         i));
+            if (isAfter) {
+                rowDataBuilder.addAfterColumns(columnBuilder.build());
+            } else {
+                rowDataBuilder.addBeforeColumns(columnBuilder.build());
+            }
+        }
+
+        return tableError;
+
+    }
+
+    private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
+        Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
+        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
+        rowChangeBuider.setSql(queryString);
+        rowChangeBuider.setEventType(EventType.QUERY);
+        return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
+    }
+
+    private Header createHeader(String binlogFile, LogHeader logHeader, String schemaName, String tableName,
+                                EventType eventType) {
+        // header会做信息冗余,方便以后做检索或者过滤
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setVersion(version);
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
+        headerBuilder.setServerId(logHeader.getServerId());
+        headerBuilder.setServerenCode(UTF_8);// 经过java输出后所有的编码为unicode
+        headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L);
+        headerBuilder.setSourceType(Type.MYSQL);
+        if (eventType != null) {
+            headerBuilder.setEventType(eventType);
+        }
+        if (schemaName != null) {
+            headerBuilder.setSchemaName(schemaName);
+        }
+        if (tableName != null) {
+            headerBuilder.setTableName(tableName);
+        }
+        headerBuilder.setEventLength(logHeader.getEventLen());
+        return headerBuilder.build();
+    }
+
+    private boolean isUpdate(List bfColumns, String newValue, int index) {
+        if (bfColumns == null) {
+            throw new CanalParseException("ERROR ## the bfColumns is null");
+        }
+
+        if (index < 0) {
+            return false;
+        }
+        if ((bfColumns.size() - 1) < index) {
+            return false;
+        }
+        Column column = bfColumns.get(index);
+
+        if (column.getIsNull()) {
+            if (newValue != null) {
+                return true;
+            }
+        } else {
+            if (newValue == null) {
+                return true;
+            } else {
+                if (!column.getValue().equals(newValue)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
+        try {
+            return tableMetaCache.getTableMeta(dbName, tbName, useCache);
+        } catch (Exception e) {
+            String message = ExceptionUtils.getRootCauseMessage(e);
+            if (filterTableError) {
+                if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
+                    return null;
+                }
+            }
+
+            throw new CanalParseException(e);
+        }
+    }
+
+    private boolean isText(String columnType) {
+        return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
+               || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
+    }
+
+    public static TransactionBegin createTransactionBegin(long threadId) {
+        TransactionBegin.Builder beginBuilder = TransactionBegin.newBuilder();
+        beginBuilder.setThreadId(threadId);
+        return beginBuilder.build();
+    }
+
+    public static TransactionEnd createTransactionEnd(long transactionId) {
+        TransactionEnd.Builder endBuilder = TransactionEnd.newBuilder();
+        endBuilder.setTransactionId(String.valueOf(transactionId));
+        return endBuilder.build();
+    }
+
+    public static Pair createSpecialPair(String key, String value) {
+        Pair.Builder pairBuilder = Pair.newBuilder();
+        pairBuilder.setKey(key);
+        pairBuilder.setValue(value);
+        return pairBuilder.build();
+    }
+
+    public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(header);
+        entryBuilder.setEntryType(entryType);
+        entryBuilder.setStoreValue(storeValue);
+        return entryBuilder.build();
+    }
+
+    public void setCharset(Charset charset) {
+        this.charset = charset;
+    }
+
+    public void setNameFilter(AviaterRegexFilter nameFilter) {
+        this.nameFilter = nameFilter;
+    }
+
+    public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
+        this.nameBlackFilter = nameBlackFilter;
+    }
+
+    public void setTableMetaCache(TableMetaCache tableMetaCache) {
+        this.tableMetaCache = tableMetaCache;
+    }
+
+    public void setFilterQueryDcl(boolean filterQueryDcl) {
+        this.filterQueryDcl = filterQueryDcl;
+    }
+
+    public void setFilterQueryDml(boolean filterQueryDml) {
+        this.filterQueryDml = filterQueryDml;
+    }
+
+    public void setFilterQueryDdl(boolean filterQueryDdl) {
+        this.filterQueryDdl = filterQueryDdl;
+    }
+
+    public void setFilterTableError(boolean filterTableError) {
+        this.filterTableError = filterTableError;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java
new file mode 100644
index 00000000..52e213e4
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/SimpleDdlParser.java
@@ -0,0 +1,275 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.oro.text.regex.Perl5Matcher;
+
+import com.alibaba.otter.canal.filter.PatternUtils;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+
+/**
+ * 简单的ddl解析工具类，后续可使用cobar/druid的SqlParser进行语法树解析
+ * 
+ * 
+ * 解析支持：
+ * a. 带schema: retl.retl_mark
+ * b. 带引号` :  `retl.retl_mark`
+ * c. 存在换行符： create table \n `retl.retl_mark`
+ * 

+ * 
+ * http://dev.mysql.com/doc/refman/5.6/en/sql-syntax-data-definition.html
+ * 
+ * @author jianghang 2013-1-22 下午10:03:22
+ * @version 1.0.0
+ */
+public class SimpleDdlParser {
+
+    public static final String CREATE_PATTERN       = "^\\s*CREATE\\s*(TEMPORARY)?\\s*TABLE\\s*(.*)$";
+    public static final String DROP_PATTERN         = "^\\s*DROP\\s*(TEMPORARY)?\\s*TABLE\\s*(.*)$";
+    public static final String ALERT_PATTERN        = "^\\s*ALTER\\s*(IGNORE)?\\s*TABLE\\s*(.*)$";
+    public static final String TRUNCATE_PATTERN     = "^\\s*TRUNCATE\\s*(TABLE)?\\s*(.*)$";
+    public static final String TABLE_PATTERN        = "^(IF\\s*NOT\\s*EXISTS\\s*)?(IF\\s*EXISTS\\s*)?(`?.+?`?[;\\(\\s]+?)?.*$"; // 采用非贪婪模式
+    public static final String INSERT_PATTERN       = "^\\s*(INSERT|MERGE|REPLACE)(.*)$";
+    public static final String UPDATE_PATTERN       = "^\\s*UPDATE(.*)$";
+    public static final String DELETE_PATTERN       = "^\\s*DELETE(.*)$";
+    public static final String RENAME_PATTERN       = "^\\s*RENAME\\s*TABLE\\s*(.*?)\\s*TO\\s*(.*?)$";
+    /**
+     * 
+     * CREATE [UNIQUE|FULLTEXT|SPATIAL] INDEX index_name
+     *         [index_type]
+     *         ON tbl_name (index_col_name,...)
+     *         [algorithm_option | lock_option] ...
+     *         
+     * http://dev.mysql.com/doc/refman/5.6/en/create-index.html
+     * 

+     */
+    public static final String CREATE_INDEX_PATTERN = "^\\s*CREATE\\s*.*?\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
+    public static final String DROP_INDEX_PATTERN   = "^\\s*DROP\\s*INDEX\\s*(.*?)\\s*ON\\s*(.*?)$";
+
+    public static DdlResult parse(String queryString, String schmeaName) {
+        queryString = removeComment(queryString); // 去除/* */的sql注释内容
+        DdlResult result = parseDdl(queryString, schmeaName, ALERT_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.ALTER);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, CREATE_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.CREATE);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, DROP_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.ERASE);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, TRUNCATE_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.TRUNCATE);
+            return result;
+        }
+
+        result = parseRename(queryString, schmeaName, RENAME_PATTERN);
+        if (result != null) {
+            result.setType(EventType.RENAME);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, CREATE_INDEX_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.CINDEX);
+            return result;
+        }
+
+        result = parseDdl(queryString, schmeaName, DROP_INDEX_PATTERN, 2);
+        if (result != null) {
+            result.setType(EventType.DINDEX);
+            return result;
+        }
+
+        result = new DdlResult(schmeaName);
+        if (isDml(queryString, INSERT_PATTERN)) {
+            result.setType(EventType.INSERT);
+            return result;
+        }
+
+        if (isDml(queryString, UPDATE_PATTERN)) {
+            result.setType(EventType.UPDATE);
+            return result;
+        }
+
+        if (isDml(queryString, DELETE_PATTERN)) {
+            result.setType(EventType.DELETE);
+            return result;
+        }
+
+        result.setType(EventType.QUERY);
+        return result;
+    }
+
+    private static DdlResult parseDdl(String queryString, String schmeaName, String pattern, int index) {
+        Perl5Matcher matcher = new Perl5Matcher();
+        if (matcher.matches(queryString, PatternUtils.getPattern(pattern))) {
+            DdlResult result = parseTableName(matcher.getMatch().group(index), schmeaName);
+            return result != null ? result : new DdlResult(schmeaName); // 无法解析时，直接返回schmea，进行兼容处理
+        }
+
+        return null;
+    }
+
+    private static boolean isDml(String queryString, String pattern) {
+        Perl5Matcher matcher = new Perl5Matcher();
+        if (matcher.matches(queryString, PatternUtils.getPattern(pattern))) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private static DdlResult parseRename(String queryString, String schmeaName, String pattern) {
+        Perl5Matcher matcher = new Perl5Matcher();
+        if (matcher.matches(queryString, PatternUtils.getPattern(pattern))) {
+            DdlResult orign = parseTableName(matcher.getMatch().group(1), schmeaName);
+            DdlResult target = parseTableName(matcher.getMatch().group(2), schmeaName);
+            if (orign != null && target != null) {
+                return new DdlResult(target.getSchemaName(),
+                    target.getTableName(),
+                    orign.getSchemaName(),
+                    orign.getTableName());
+            }
+        }
+
+        return null;
+    }
+
+    private static DdlResult parseTableName(String matchString, String schmeaName) {
+        Perl5Matcher tableMatcher = new Perl5Matcher();
+        matchString = matchString + " ";
+        if (tableMatcher.matches(matchString, PatternUtils.getPattern(TABLE_PATTERN))) {
+            String tableString = tableMatcher.getMatch().group(3);
+
+            tableString = StringUtils.removeEnd(tableString, ";");
+            tableString = StringUtils.removeEnd(tableString, "(");
+            tableString = StringUtils.trim(tableString);
+            // 特殊处理引号`
+            tableString = removeEscape(tableString);
+            // 处理schema.table的写法
+            String names[] = StringUtils.split(tableString, ".");
+            if (names != null && names.length > 1) {
+                return new DdlResult(removeEscape(names[0]), removeEscape(names[1]));
+            } else {
+                return new DdlResult(schmeaName, removeEscape(names[0]));
+            }
+        }
+
+        return null;
+    }
+
+    private static String removeEscape(String str) {
+        String result = StringUtils.removeEnd(str, "`");
+        result = StringUtils.removeStart(result, "`");
+        return result;
+    }
+
+    private static String removeComment(String sql) {
+        if (sql == null) {
+            return null;
+        }
+
+        String start = "/*";
+        String end = "*/";
+        while (true) {
+            // 循环找到所有的注释
+            int index0 = sql.indexOf(start);
+            if (index0 == -1) {
+                return sql;
+            }
+            int index1 = sql.indexOf(end, index0);
+            if (index1 == -1) {
+                return sql;
+            }
+            StringBuilder sb = new StringBuilder();
+            sb.append(sql.substring(0, index0));
+            sb.append(" ");
+            sb.append(sql.substring(index1 + end.length()));
+            sql = sb.toString();
+        }
+    }
+
+    public static class DdlResult {
+
+        private String    schemaName;
+        private String    tableName;
+        private String    oriSchemaName; // rename ddl中的源表
+        private String    oriTableName; // rename ddl中的目标表
+        private EventType type;
+
+        public DdlResult(){
+        }
+
+        public DdlResult(String schemaName){
+            this.schemaName = schemaName;
+        }
+
+        public DdlResult(String schemaName, String tableName){
+            this.schemaName = schemaName;
+            this.tableName = tableName;
+        }
+
+        public DdlResult(String schemaName, String tableName, String oriSchemaName, String oriTableName){
+            this.schemaName = schemaName;
+            this.tableName = tableName;
+            this.oriSchemaName = oriSchemaName;
+            this.oriTableName = oriTableName;
+        }
+
+        public String getSchemaName() {
+            return schemaName;
+        }
+
+        public void setSchemaName(String schemaName) {
+            this.schemaName = schemaName;
+        }
+
+        public String getTableName() {
+            return tableName;
+        }
+
+        public void setTableName(String tableName) {
+            this.tableName = tableName;
+        }
+
+        public EventType getType() {
+            return type;
+        }
+
+        public void setType(EventType type) {
+            this.type = type;
+        }
+
+        public String getOriSchemaName() {
+            return oriSchemaName;
+        }
+
+        public void setOriSchemaName(String oriSchemaName) {
+            this.oriSchemaName = oriSchemaName;
+        }
+
+        public String getOriTableName() {
+            return oriTableName;
+        }
+
+        public void setOriTableName(String oriTableName) {
+            this.oriTableName = oriTableName;
+        }
+
+        @Override
+        public String toString() {
+            return "DdlResult [schemaName=" + schemaName + ", tableName=" + tableName + ", oriSchemaName="
+                   + oriSchemaName + ", oriTableName=" + oriTableName + ", type=" + type + "]";
+        }
+
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
new file mode 100644
index 00000000..f731ff0c
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
@@ -0,0 +1,139 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.FieldPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.TableMeta;
+import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 处理table meta解析和缓存
+ * 
+ * @author jianghang 2013-1-17 下午10:15:16
+ * @version 1.0.0
+ */
+public class TableMetaCache {
+
+    public static final String     COLUMN_NAME    = "COLUMN_NAME";
+    public static final String     COLUMN_TYPE    = "COLUMN_TYPE";
+    public static final String     IS_NULLABLE    = "IS_NULLABLE";
+    public static final String     COLUMN_KEY     = "COLUMN_KEY";
+    public static final String     COLUMN_DEFAULT = "COLUMN_DEFAULT";
+    public static final String     EXTRA          = "EXTRA";
+    private MysqlConnection        connection;
+
+    // 第一层tableId,第二层schema.table,解决tableId重复，对应多张表
+    private Map tableMetaCache;
+
+    public TableMetaCache(MysqlConnection con){
+        this.connection = con;
+        tableMetaCache = new MapMaker().makeComputingMap(new Function() {
+
+            public TableMeta apply(String name) {
+                try {
+                    return getTableMeta0(name);
+                } catch (IOException e) {
+                    // 尝试做一次retry操作
+                    try {
+                        connection.reconnect();
+                        return getTableMeta0(name);
+                    } catch (IOException e1) {
+                        throw new CanalParseException("fetch failed by table meta:" + name, e1);
+                    }
+                }
+            }
+
+        });
+
+    }
+
+    public TableMeta getTableMeta(String schema, String table) {
+        return getTableMeta(schema, table, true);
+    }
+
+    public TableMeta getTableMeta(String schema, String table, boolean useCache) {
+        if (!useCache) {
+            tableMetaCache.remove(getFullName(schema, table));
+        }
+
+        return tableMetaCache.get(getFullName(schema, table));
+    }
+
+    public void clearTableMeta(String schema, String table) {
+        tableMetaCache.remove(getFullName(schema, table));
+    }
+
+    public void clearTableMetaWithSchemaName(String schema) {
+        // Set removeNames = new HashSet(); //
+        // 存一份临时变量，避免在遍历的时候进行删除
+        for (String name : tableMetaCache.keySet()) {
+            if (StringUtils.startsWithIgnoreCase(name, schema + ".")) {
+                // removeNames.add(name);
+                tableMetaCache.remove(name);
+            }
+        }
+
+        // for (String name : removeNames) {
+        // tables.remove(name);
+        // }
+    }
+
+    public void clearTableMeta() {
+        tableMetaCache.clear();
+    }
+
+    private TableMeta getTableMeta0(String fullname) throws IOException {
+        ResultSetPacket packet = connection.query("desc " + fullname);
+        return new TableMeta(fullname, parserTableMeta(packet));
+    }
+
+    private List parserTableMeta(ResultSetPacket packet) {
+        Map nameMaps = new HashMap(6, 1f);
+
+        int index = 0;
+        for (FieldPacket fieldPacket : packet.getFieldDescriptors()) {
+            nameMaps.put(fieldPacket.getOriginalName(), index++);
+        }
+
+        int size = packet.getFieldDescriptors().size();
+        int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size();
+        List result = new ArrayList();
+        for (int i = 0; i < count; i++) {
+            FieldMeta meta = new FieldMeta();
+            // 做一个优化，使用String.intern()，共享String对象，减少内存使用
+            meta.setColumnName(packet.getFieldValues().get(nameMaps.get(COLUMN_NAME) + i * size).intern());
+            meta.setColumnType(packet.getFieldValues().get(nameMaps.get(COLUMN_TYPE) + i * size));
+            meta.setIsNullable(packet.getFieldValues().get(nameMaps.get(IS_NULLABLE) + i * size));
+            meta.setIskey(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size));
+            meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size));
+            meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size));
+
+            result.add(meta);
+        }
+
+        return result;
+    }
+
+    private String getFullName(String schema, String table) {
+        StringBuilder builder = new StringBuilder();
+        return builder.append('`')
+            .append(schema)
+            .append('`')
+            .append('.')
+            .append('`')
+            .append(table)
+            .append('`')
+            .toString();
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java
new file mode 100644
index 00000000..a5a5dadb
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BinLogFileQueue.java
@@ -0,0 +1,225 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.local;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+
+/**
+ * 维护binlog文件列表
+ * 
+ * @author jianghang 2012-7-7 下午03:48:05
+ * @version 1.0.0
+ */
+public class BinLogFileQueue {
+
+    private String        baseName       = "mysql-bin.";
+    private List    binlogs        = new ArrayList();
+    private File          directory;
+    private ReentrantLock lock           = new ReentrantLock();
+    private Condition     nextCondition  = lock.newCondition();
+    private Timer         timer          = new Timer(true);
+    private long          reloadInterval = 10 * 1000L;           // 10秒
+
+    public BinLogFileQueue(String directory){
+        this(new File(directory));
+    }
+
+    public BinLogFileQueue(File directory){
+        this.directory = directory;
+
+        if (!directory.canRead()) {
+            throw new CanalParseException("Binlog index missing or unreadable;  " + directory.getAbsolutePath());
+        }
+
+        List files = listBinlogFiles();
+        for (File file : files) {
+            offer(file);
+        }
+
+        timer.scheduleAtFixedRate(new TimerTask() {
+
+            public void run() {
+                List files = listBinlogFiles();
+                for (File file : files) {
+                    offer(file);
+                }
+            }
+        }, reloadInterval, reloadInterval);
+    }
+
+    /**
+     * 根据前一个文件，获取符合条件的下一个binlog文件
+     * 
+     * @param pre
+     * @return
+     */
+    public File getNextFile(File pre) {
+        try {
+            lock.lockInterruptibly();
+            if (binlogs.size() == 0) {
+                return null;
+            } else {
+                if (pre == null) {// 第一次
+                    return binlogs.get(0);
+                } else {
+                    int index = seek(pre);
+                    if (index < binlogs.size() - 1) {
+                        return binlogs.get(index + 1);
+                    } else {
+                        return null;
+                    }
+                }
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return null;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public File getBefore(File file) {
+        try {
+            lock.lockInterruptibly();
+            if (binlogs.size() == 0) {
+                return null;
+            } else {
+                if (file == null) {// 第一次
+                    return binlogs.get(binlogs.size() - 1);
+                } else {
+                    int index = seek(file);
+                    if (index > 0) {
+                        return binlogs.get(index - 1);
+                    } else {
+                        return null;
+                    }
+                }
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return null;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * 根据前一个文件，获取符合条件的下一个binlog文件
+     * 
+     * @param pre
+     * @return
+     * @throws InterruptedException
+     */
+    public File waitForNextFile(File pre) throws InterruptedException {
+        try {
+            lock.lockInterruptibly();
+            if (binlogs.size() == 0) {
+                nextCondition.await();// 等待新文件
+            }
+
+            if (pre == null) {// 第一次
+                return binlogs.get(0);
+            } else {
+                int index = seek(pre);
+                if (index < binlogs.size() - 1) {
+                    return binlogs.get(index + 1);
+                } else {
+                    nextCondition.await();// 等待新文件
+                    return waitForNextFile(pre);// 唤醒之后递归调用一下
+                }
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * 获取当前所有binlog文件
+     */
+    public List currentBinlogs() {
+        return new ArrayList(binlogs);
+    }
+
+    public void destory() {
+        try {
+            lock.lockInterruptibly();
+            timer.cancel();
+            binlogs.clear();
+
+            nextCondition.signalAll();// 唤醒线程，通知退出
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private boolean offer(File file) {
+        try {
+            lock.lockInterruptibly();
+            if (!binlogs.contains(file)) {
+                binlogs.add(file);
+                nextCondition.signalAll();// 唤醒
+                return true;
+            } else {
+                return false;
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            return false;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private List listBinlogFiles() {
+        List files = new ArrayList();
+        files.addAll(FileUtils.listFiles(directory, new IOFileFilter() {
+
+            public boolean accept(File file) {
+                return file.getName().startsWith(baseName);
+            }
+
+            public boolean accept(File dir, String name) {
+                return true;
+            }
+        }, null));
+        // 排一下序列
+        Collections.sort(files, new Comparator() {
+
+            public int compare(File o1, File o2) {
+                return o1.getName().compareTo(o2.getName());
+            }
+
+        });
+        return files;
+    }
+
+    private int seek(File file) {
+        for (int i = 0; i < binlogs.size(); i++) {
+            File binlog = binlogs.get(i);
+            if (binlog.getName().equals(file.getName())) {
+                return i;
+            }
+        }
+
+        return -1;
+    }
+
+    // ================== setter / getter ===================
+
+    public void setBaseName(String baseName) {
+        this.baseName = baseName;
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BufferedFileDataInput.java b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BufferedFileDataInput.java
new file mode 100644
index 00000000..aade41e7
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/local/BufferedFileDataInput.java
@@ -0,0 +1,92 @@
+package com.alibaba.otter.canal.parse.inbound.mysql.local;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.FileChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author jianghang 2012-7-7 下午03:10:47
+ * @version 1.0.0
+ */
+public class BufferedFileDataInput {
+
+    private static final Logger logger = LoggerFactory.getLogger(BufferedFileDataInput.class);
+    // Read parameters.
+    private File                file;
+    private int                 size;
+
+    // Variables to control reading.
+    private FileInputStream     fileInput;
+    private BufferedInputStream bufferedInput;
+    private DataInputStream     dataInput;
+    private long                offset;
+    private FileChannel         fileChannel;
+
+    public BufferedFileDataInput(File file, int size) throws FileNotFoundException, IOException, InterruptedException{
+        this.file = file;
+        this.size = size;
+    }
+
+    public BufferedFileDataInput(File file) throws FileNotFoundException, IOException, InterruptedException{
+        this(file, 1024);
+    }
+
+    public long available() throws IOException {
+        return fileChannel.size() - offset;
+    }
+
+    public long skip(long bytes) throws IOException {
+        long bytesSkipped = bufferedInput.skip(bytes);
+        offset += bytesSkipped;
+        return bytesSkipped;
+    }
+
+    public void seek(long seekBytes) throws FileNotFoundException, IOException, InterruptedException {
+        fileInput = new FileInputStream(file);
+        fileChannel = fileInput.getChannel();
+
+        try {
+            fileChannel.position(seekBytes);
+        } catch (ClosedByInterruptException e) {
+            throw new InterruptedException();
+        }
+        bufferedInput = new BufferedInputStream(fileInput, size);
+        dataInput = new DataInputStream(bufferedInput);
+        offset = seekBytes;
+    }
+
+    public void readFully(byte[] bytes) throws IOException {
+        readFully(bytes, 0, bytes.length);
+    }
+
+    public void readFully(byte[] bytes, int start, int len) throws IOException {
+        dataInput.readFully(bytes, start, len);
+        offset += len;
+    }
+
+    public void close() {
+        try {
+            if (fileChannel != null) {
+                fileChannel.close();
+                fileInput.close();
+            }
+        } catch (IOException e) {
+            logger.warn("Unable to close buffered file reader: file=" + file.getName() + " exception=" + e.getMessage());
+        }
+
+        fileChannel = null;
+        fileInput = null;
+        bufferedInput = null;
+        dataInput = null;
+        offset = -1;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java
new file mode 100644
index 00000000..25f63d15
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/CanalLogPositionManager.java
@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.parse.index;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 接口组合
+ * 
+ * @author jianghang 2012-7-7 上午10:02:02
+ * @version 1.0.0
+ */
+public interface CanalLogPositionManager extends CanalLifeCycle {
+
+    LogPosition getLatestIndexBy(String destination);
+
+    void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException;
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/FailbackLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FailbackLogPositionManager.java
new file mode 100644
index 00000000..6fa8efa5
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FailbackLogPositionManager.java
@@ -0,0 +1,74 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 实现基于failover查找的机制完成meta的操作
+ * 
+ * 
+ * 应用场景：比如针对内存buffer，出现HA切换，先尝试从内存buffer区中找到lastest position，如果不存在才尝试找一下meta里消费的信息
+ * 

+ * 
+ * @author jianghang 2012-7-20 下午02:33:20
+ */
+public class FailbackLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    private CanalLogPositionManager primary;
+    private CanalLogPositionManager failback;
+
+    public void start() {
+        super.start();
+        Assert.notNull(primary);
+        Assert.notNull(failback);
+
+        if (!primary.isStart()) {
+            primary.start();
+        }
+
+        if (!failback.isStart()) {
+            failback.start();
+        }
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (primary.isStart()) {
+            primary.stop();
+        }
+
+        if (failback.isStart()) {
+            failback.stop();
+        }
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPosition = primary.getLatestIndexBy(destination);
+        if (logPosition == null) {
+            return failback.getLatestIndexBy(destination);
+        } else {
+            return logPosition;
+        }
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
+        try {
+            primary.persistLogPosition(destination, logPosition);
+        } catch (CanalParseException e) {
+            failback.persistLogPosition(destination, logPosition);
+        }
+    }
+
+    public void setPrimary(CanalLogPositionManager primary) {
+        this.primary = primary;
+    }
+
+    public void setFailback(CanalLogPositionManager failback) {
+        this.failback = failback;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
new file mode 100644
index 00000000..ee1e178b
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
@@ -0,0 +1,194 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于文件刷新的log position实现
+ * 
+ * 
+ * 策略：
+ * 1. 先写内存，然后定时刷新数据到File
+ * 2. 数据采取overwrite模式(只保留最后一次)
+ * 

+ * 
+ * @author jianghang 2013-4-15 下午09:40:48
+ * @version 1.0.4
+ */
+public class FileMixedLogPositionManager extends MemoryLogPositionManager {
+
+    private static final Logger      logger       = LoggerFactory.getLogger(FileMixedLogPositionManager.class);
+    private static final Charset     charset      = Charset.forName("UTF-8");
+    private File                     dataDir;
+    private String                   dataFileName = "parse.dat";
+    private Map        dataFileCaches;
+    private ScheduledExecutorService executor;
+    @SuppressWarnings("serial")
+    private final LogPosition        nullPosition = new LogPosition() {
+                                                  };
+
+    private long                     period       = 1000;                                                      // 单位ms
+    private Set              persistTasks;
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(dataDir);
+        if (!dataDir.exists()) {
+            try {
+                FileUtils.forceMkdir(dataDir);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+
+        if (!dataDir.canRead() || !dataDir.canWrite()) {
+            throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
+        }
+
+        dataFileCaches = new MapMaker().makeComputingMap(new Function() {
+
+            public File apply(String destination) {
+                return getDataFile(destination);
+            }
+        });
+
+        executor = Executors.newScheduledThreadPool(1);
+        positions = new MapMaker().makeComputingMap(new Function() {
+
+            public LogPosition apply(String destination) {
+                LogPosition logPosition = loadDataFromFile(dataFileCaches.get(destination));
+                if (logPosition == null) {
+                    return nullPosition;
+                } else {
+                    return logPosition;
+                }
+            }
+        });
+
+        persistTasks = Collections.synchronizedSet(new HashSet());
+
+        // 启动定时工作任务
+        executor.scheduleAtFixedRate(new Runnable() {
+
+            public void run() {
+                List tasks = new ArrayList(persistTasks);
+                for (String destination : tasks) {
+                    try {
+                        // 定时将内存中的最新值刷到file中，多次变更只刷一次
+                        flushDataToFile(destination);
+                        persistTasks.remove(destination);
+                    } catch (Throwable e) {
+                        // ignore
+                        logger.error("period update" + destination + " curosr failed!", e);
+                    }
+                }
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    public void stop() {
+        super.stop();
+
+        flushDataToFile();
+        executor.shutdownNow();
+        positions.clear();
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        persistTasks.add(destination);// 添加到任务队列中进行触发
+        super.persistLogPosition(destination, logPosition);
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPostion = super.getLatestIndexBy(destination);
+        if (logPostion == nullPosition) {
+            return null;
+        } else {
+            return logPostion;
+        }
+    }
+
+    // ============================ helper method ======================
+
+    private File getDataFile(String destination) {
+        File destinationMetaDir = new File(dataDir, destination);
+        if (!destinationMetaDir.exists()) {
+            try {
+                FileUtils.forceMkdir(destinationMetaDir);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+
+        return new File(destinationMetaDir, dataFileName);
+    }
+
+    private void flushDataToFile() {
+        for (String destination : positions.keySet()) {
+            flushDataToFile(destination);
+        }
+    }
+
+    private void flushDataToFile(String destination) {
+        flushDataToFile(destination, dataFileCaches.get(destination));
+    }
+
+    private void flushDataToFile(String destination, File dataFile) {
+        LogPosition position = positions.get(destination);
+        if (position != null && position != nullPosition) {
+            String json = JsonUtils.marshalToString(position);
+            try {
+                FileUtils.writeStringToFile(dataFile, json);
+            } catch (IOException e) {
+                throw new CanalMetaManagerException(e);
+            }
+        }
+    }
+
+    private LogPosition loadDataFromFile(File dataFile) {
+        try {
+            if (!dataFile.exists()) {
+                return null;
+            }
+
+            String json = FileUtils.readFileToString(dataFile, charset.name());
+            return JsonUtils.unmarshalFromString(json, LogPosition.class);
+        } catch (IOException e) {
+            throw new CanalMetaManagerException(e);
+        }
+    }
+
+    public void setDataDir(String dataDir) {
+        this.dataDir = new File(dataDir);
+    }
+
+    public void setDataDir(File dataDir) {
+        this.dataDir = dataDir;
+    }
+
+    public void setPeriod(long period) {
+        this.period = period;
+    }
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java
new file mode 100644
index 00000000..c95fa4e2
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManager.java
@@ -0,0 +1,39 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.Map;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于内存的实现
+ * 
+ * @author jianghang 2012-7-7 上午10:17:23
+ * @version 1.0.0
+ */
+public class MemoryLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    protected Map positions;
+
+    public void start() {
+        super.start();
+
+        positions = new MapMaker().makeMap();
+    }
+
+    public void stop() {
+        super.stop();
+
+        positions.clear();
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        return positions.get(destination);
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        positions.put(destination, logPosition);
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManager.java
new file mode 100644
index 00000000..3af97411
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManager.java
@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.store.helper.CanalEventUtils;
+
+/**
+ * 基于meta信息的实现
+ * 
+ * @author jianghang 2012-7-10 下午05:02:33
+ * @version 1.0.0
+ */
+public class MetaLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    private static final Logger logger = LoggerFactory.getLogger(MetaLogPositionManager.class);
+    private CanalMetaManager    metaManager;
+
+    public void start() {
+        super.start();
+        Assert.notNull(metaManager);
+        if (!metaManager.isStart()) {
+            metaManager.start();
+        }
+    }
+
+    public void stop() {
+        super.stop();
+        if (metaManager.isStart()) {
+            metaManager.stop();
+        }
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        // do nothing
+        logger.info("persist LogPosition:{}", destination, logPosition);
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        List clientIdentitys = metaManager.listAllSubscribeInfo(destination);
+        LogPosition result = null;
+        if (!CollectionUtils.isEmpty(clientIdentitys)) {
+            // 尝试找到一个最小的logPosition
+            for (ClientIdentity clientIdentity : clientIdentitys) {
+                LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
+                if (position == null) {
+                    continue;
+                }
+
+                if (result == null) {
+                    result = position;
+                } else {
+                    result = CanalEventUtils.min(result, position);
+                }
+            }
+        }
+
+        return result;
+    }
+
+    public void setMetaManager(CanalMetaManager metaManager) {
+        this.metaManager = metaManager;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
new file mode 100644
index 00000000..6dfc7c76
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
@@ -0,0 +1,90 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 混合memory + zookeeper的存储模式
+ * 
+ * @author jianghang 2012-7-7 上午10:33:19
+ * @version 1.0.0
+ */
+public class MixedLogPositionManager extends MemoryLogPositionManager implements CanalLogPositionManager {
+
+    private static final Logger         logger       = LoggerFactory.getLogger(MixedLogPositionManager.class);
+    private ZooKeeperLogPositionManager zooKeeperLogPositionManager;
+    private ExecutorService             executor;
+    @SuppressWarnings("serial")
+    private final LogPosition           nullPosition = new LogPosition() {
+                                                     };
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(zooKeeperLogPositionManager);
+        if (!zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.start();
+        }
+        executor = Executors.newFixedThreadPool(1);
+        positions = new MapMaker().makeComputingMap(new Function() {
+
+            public LogPosition apply(String destination) {
+                LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
+                if (logPosition == null) {
+                    return nullPosition;
+                } else {
+                    return logPosition;
+                }
+            }
+        });
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.stop();
+        }
+        executor.shutdownNow();
+        positions.clear();
+    }
+
+    public void persistLogPosition(final String destination, final LogPosition logPosition) {
+        super.persistLogPosition(destination, logPosition);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                try {
+                    zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
+                } catch (Exception e) {
+                    logger.error("ERROR # persist to zookeepr has an error", e);
+                }
+            }
+        });
+
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPosition = super.getLatestIndexBy(destination);
+        if (logPosition == nullPosition) {
+            return null;
+        } else {
+            return logPosition;
+        }
+    }
+
+    // ======================== setter / getter ======================
+
+    public void setZooKeeperLogPositionManager(ZooKeeperLogPositionManager zooKeeperLogPositionManager) {
+        this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java
new file mode 100644
index 00000000..b1967163
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java
@@ -0,0 +1,111 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 基于定时刷新的策略的mixed实现
+ * 
+ * @author jianghang 2012-9-12 上午11:18:14
+ * @version 1.0.0
+ */
+public class PeriodMixedLogPositionManager extends MemoryLogPositionManager implements CanalLogPositionManager {
+
+    private static final Logger         logger       = LoggerFactory.getLogger(PeriodMixedLogPositionManager.class);
+    private ZooKeeperLogPositionManager zooKeeperLogPositionManager;
+    private ScheduledExecutorService    executor;
+    @SuppressWarnings("serial")
+    private final LogPosition           nullPosition = new LogPosition() {
+                                                     };
+
+    private long                        period       = 1000;                                                        // 单位ms
+    private Set                 persistTasks;
+
+    public void start() {
+        super.start();
+
+        Assert.notNull(zooKeeperLogPositionManager);
+        if (!zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.start();
+        }
+        executor = Executors.newScheduledThreadPool(1);
+        positions = new MapMaker().makeComputingMap(new Function() {
+
+            public LogPosition apply(String destination) {
+                LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
+                if (logPosition == null) {
+                    return nullPosition;
+                } else {
+                    return logPosition;
+                }
+            }
+        });
+
+        persistTasks = Collections.synchronizedSet(new HashSet());
+
+        // 启动定时工作任务
+        executor.scheduleAtFixedRate(new Runnable() {
+
+            public void run() {
+                List tasks = new ArrayList(persistTasks);
+                for (String destination : tasks) {
+                    try {
+                        // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
+                        zooKeeperLogPositionManager.persistLogPosition(destination, getLatestIndexBy(destination));
+                        persistTasks.remove(destination);
+                    } catch (Throwable e) {
+                        // ignore
+                        logger.error("period update" + destination + " curosr failed!", e);
+                    }
+                }
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zooKeeperLogPositionManager.isStart()) {
+            zooKeeperLogPositionManager.stop();
+        }
+        executor.shutdownNow();
+        positions.clear();
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        persistTasks.add(destination);// 添加到任务队列中进行触发
+        super.persistLogPosition(destination, logPosition);
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        LogPosition logPostion = super.getLatestIndexBy(destination);
+        if (logPostion == nullPosition) {
+            return null;
+        } else {
+            return logPostion;
+        }
+    }
+
+    public void setZooKeeperLogPositionManager(ZooKeeperLogPositionManager zooKeeperLogPositionManager) {
+        this.zooKeeperLogPositionManager = zooKeeperLogPositionManager;
+    }
+
+    public void setPeriod(long period) {
+        this.period = period;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManager.java b/parse/src/main/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManager.java
new file mode 100644
index 00000000..996297ac
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManager.java
@@ -0,0 +1,57 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+/**
+ * 基于zk的实现
+ * 
+ * @author jianghang 2012-7-7 上午10:08:27
+ * @version 1.0.0
+ */
+public class ZooKeeperLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+    private ZkClientx zkClientx;
+
+    public void start() {
+        super.start();
+        Assert.notNull(zkClientx);
+    }
+
+    public void stop() {
+        super.stop();
+    }
+
+    public LogPosition getLatestIndexBy(String destination) {
+        String path = ZookeeperPathUtils.getParsePath(destination);
+        byte[] data = zkClientx.readData(path, true);
+        if (data == null || data.length == 0) {
+            return null;
+        }
+
+        return JsonUtils.unmarshalFromByte(data, LogPosition.class);
+    }
+
+    public void persistLogPosition(String destination, LogPosition logPosition) {
+        String path = ZookeeperPathUtils.getParsePath(destination);
+        byte[] data = JsonUtils.marshalToByte(logPosition);
+        try {
+            zkClientx.writeData(path, data);
+        } catch (ZkNoNodeException e) {
+            zkClientx.createPersistent(path, data, true);
+        }
+    }
+
+    // ================== setter / getter =================
+
+    public void setZkClientx(ZkClientx zkClientx) {
+        this.zkClientx = zkClientx;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java b/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java
new file mode 100644
index 00000000..5446d068
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/support/AuthenticationInfo.java
@@ -0,0 +1,127 @@
+package com.alibaba.otter.canal.parse.support;
+
+import java.net.InetSocketAddress;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
+
+/**
+ * 数据库认证信息
+ * 
+ * @author jianghang 2012-7-11 上午11:22:19
+ * @version 1.0.0
+ */
+public class AuthenticationInfo {
+
+    private InetSocketAddress address;            // 主库信息
+    private String            username;           // 帐号
+    private String            password;           // 密码
+    private String            defaultDatabaseName; // 默认链接的数据库
+
+    public AuthenticationInfo(){
+        super();
+    }
+
+    public AuthenticationInfo(InetSocketAddress address, String username, String password){
+        this(address, username, password, "");
+    }
+
+    public AuthenticationInfo(InetSocketAddress address, String username, String password, String defaultDatabaseName){
+        this.address = address;
+        this.username = username;
+        this.password = password;
+        this.defaultDatabaseName = defaultDatabaseName;
+    }
+
+    public InetSocketAddress getAddress() {
+        return address;
+    }
+
+    public void setAddress(InetSocketAddress address) {
+        this.address = address;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public String getDefaultDatabaseName() {
+        return defaultDatabaseName;
+    }
+
+    public void setDefaultDatabaseName(String defaultDatabaseName) {
+        this.defaultDatabaseName = defaultDatabaseName;
+    }
+
+    @Override
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, ToStringStyle.DEFAULT_STYLE);
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((address == null) ? 0 : address.hashCode());
+        result = prime * result + ((defaultDatabaseName == null) ? 0 : defaultDatabaseName.hashCode());
+        result = prime * result + ((password == null) ? 0 : password.hashCode());
+        result = prime * result + ((username == null) ? 0 : username.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof AuthenticationInfo)) {
+            return false;
+        }
+        AuthenticationInfo other = (AuthenticationInfo) obj;
+        if (address == null) {
+            if (other.address != null) {
+                return false;
+            }
+        } else if (!address.equals(other.address)) {
+            return false;
+        }
+        if (defaultDatabaseName == null) {
+            if (other.defaultDatabaseName != null) {
+                return false;
+            }
+        } else if (!defaultDatabaseName.equals(other.defaultDatabaseName)) {
+            return false;
+        }
+        if (password == null) {
+            if (other.password != null) {
+                return false;
+            }
+        } else if (!password.equals(other.password)) {
+            return false;
+        }
+        if (username == null) {
+            if (other.username != null) {
+                return false;
+            }
+        } else if (!username.equals(other.username)) {
+            return false;
+        }
+        return true;
+    }
+
+}
diff --git a/parse/src/main/java/com/alibaba/otter/canal/parse/support/HaAuthenticationInfo.java b/parse/src/main/java/com/alibaba/otter/canal/parse/support/HaAuthenticationInfo.java
new file mode 100644
index 00000000..f9109d6e
--- /dev/null
+++ b/parse/src/main/java/com/alibaba/otter/canal/parse/support/HaAuthenticationInfo.java
@@ -0,0 +1,36 @@
+package com.alibaba.otter.canal.parse.support;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * @author zebin.xuzb 2012-9-26 下午3:08:11
+ * @version 1.0.0
+ */
+public class HaAuthenticationInfo {
+
+    private AuthenticationInfo       master;
+    private List slavers = new ArrayList();
+
+    public AuthenticationInfo getMaster() {
+        return master;
+    }
+
+    public void setMaster(AuthenticationInfo master) {
+        this.master = master;
+    }
+
+    public List getSlavers() {
+        return slavers;
+    }
+
+    public void addSlaver(AuthenticationInfo slaver) {
+        this.slavers.add(slaver);
+    }
+
+    public void addSlavers(Collection slavers) {
+        this.slavers.addAll(slavers);
+    }
+
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
new file mode 100644
index 00000000..234c1e58
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
@@ -0,0 +1,101 @@
+package com.alibaba.otter.canal.parse;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.driver.mysql.MysqlConnector;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket;
+import com.alibaba.otter.canal.parse.driver.mysql.utils.PacketManager;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.DirectLogFetcher;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogDecoder;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class DirectLogFetcherTest {
+
+    @Test
+    public void testSimple() {
+        DirectLogFetcher fetcher = new DirectLogFetcher();
+        try {
+            MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
+            connector.connect();
+            sendBinlogDump(connector, "mysql-bin.001016", 4L, 3);
+
+            fetcher.start(connector.getChannel());
+
+            LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+            LogContext context = new LogContext();
+            while (fetcher.fetch()) {
+                LogEvent event = null;
+                event = decoder.decode(fetcher, context);
+
+                if (event == null) {
+                    throw new RuntimeException("parse failed");
+                }
+
+                int eventType = event.getHeader().getType();
+                switch (eventType) {
+                    case LogEvent.ROTATE_EVENT:
+                        // binlogFileName = ((RotateLogEvent)
+                        // event).getFilename();
+                        break;
+                    case LogEvent.WRITE_ROWS_EVENT_V1:
+                    case LogEvent.WRITE_ROWS_EVENT:
+                        // parseRowsEvent((WriteRowsLogEvent) event);
+                        break;
+                    case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.UPDATE_ROWS_EVENT:
+                        // parseRowsEvent((UpdateRowsLogEvent) event);
+                        break;
+                    case LogEvent.DELETE_ROWS_EVENT_V1:
+                    case LogEvent.DELETE_ROWS_EVENT:
+                        // parseRowsEvent((DeleteRowsLogEvent) event);
+                        break;
+                    case LogEvent.QUERY_EVENT:
+                        // parseQueryEvent((QueryLogEvent) event);
+                        break;
+                    case LogEvent.ROWS_QUERY_LOG_EVENT:
+                        // parseRowsQueryEvent((RowsQueryLogEvent) event);
+                        break;
+                    case LogEvent.ANNOTATE_ROWS_EVENT:
+                        break;
+                    case LogEvent.XID_EVENT:
+                        break;
+                    default:
+                        break;
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                fetcher.close();
+            } catch (IOException e) {
+                Assert.fail(e.getMessage());
+            }
+        }
+
+    }
+
+    private void sendBinlogDump(MysqlConnector connector, String binlogfilename, Long binlogPosition, int slaveId)
+                                                                                                                  throws IOException {
+        BinlogDumpCommandPacket binlogDumpCmd = new BinlogDumpCommandPacket();
+        binlogDumpCmd.binlogFileName = binlogfilename;
+        binlogDumpCmd.binlogPosition = binlogPosition;
+        binlogDumpCmd.slaveServerId = slaveId;
+        byte[] cmdBody = binlogDumpCmd.toBytes();
+
+        HeaderPacket binlogDumpHeader = new HeaderPacket();
+        binlogDumpHeader.setPacketBodyLength(cmdBody.length);
+        binlogDumpHeader.setPacketSequenceNumber((byte) 0x00);
+        PacketManager.write(connector.getChannel(), new ByteBuffer[] { ByteBuffer.wrap(binlogDumpHeader.toBytes()),
+                ByteBuffer.wrap(cmdBody) });
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java b/parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java
new file mode 100644
index 00000000..d9263d25
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/helper/TimeoutChecker.java
@@ -0,0 +1,59 @@
+package com.alibaba.otter.canal.parse.helper;
+
+/**
+ * 用于检查超时, 主要用于启动服务以后，如果在指定的时间内没有响应，则自动退出
+ * 
+ * @author: yuanzu Date: 12-9-26 Time: 上午10:55
+ */
+public class TimeoutChecker {
+
+    /**
+     * 最后一次动作的时间
+     */
+    private long              lastTouch;
+    /**
+     * 超时时间
+     */
+    private long              timeoutMillis;
+
+    /**
+     * 运行标志
+     */
+    private boolean           running                = true;
+
+    /**
+     * default 3s
+     */
+    private static final long DEFAULT_TIMEOUT_MILLIS = 3 * 1000;
+
+    public TimeoutChecker(long timeoutMillis){
+        this.timeoutMillis = timeoutMillis;
+        touch();
+    }
+
+    public TimeoutChecker(){
+        this(DEFAULT_TIMEOUT_MILLIS);
+    }
+
+    /**
+     * 更新
+     */
+    public void touch() {
+        this.lastTouch = System.currentTimeMillis();
+    }
+
+    /**
+     * 等待空闲
+     * 
+     * @throws InterruptedException
+     */
+    public void waitForIdle() throws InterruptedException {
+        while (this.running && (System.currentTimeMillis() - this.lastTouch) < this.timeoutMillis) {
+            Thread.sleep(50);
+        }
+    }
+
+    public void stop() {
+        this.running = false;
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java
new file mode 100644
index 00000000..7565b025
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/EventTransactionBufferTest.java
@@ -0,0 +1,138 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.inbound.EventTransactionBuffer.TransactionFlushCallback;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+
+public class EventTransactionBufferTest {
+
+    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+    private static final String messgae     = "{0} [{1}:{2}:{3}] {4}.{5}";
+
+    @Test
+    public void testTransactionFlush() {
+        final int bufferSize = 64;
+        final int transactionSize = 5;
+        EventTransactionBuffer buffer = new EventTransactionBuffer();
+        buffer.setBufferSize(bufferSize);
+        buffer.setFlushCallback(new TransactionFlushCallback() {
+
+            public void flush(List transaction) throws InterruptedException {
+                Assert.assertEquals(transactionSize, transaction.size());
+                System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+                for (Entry data : transaction) {
+
+                    CanalEntry.Header header = data.getHeader();
+                    Date date = new Date(header.getExecuteTime());
+                    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+                    if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || data.getEntryType() == EntryType.TRANSACTIONEND) {
+                        System.out.println(data.getEntryType());
+
+                    } else {
+                        System.out.println(MessageFormat.format(messgae, new Object[] {
+                                Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(),
+                                format.format(date), header.getSchemaName(), header.getTableName() }));
+                    }
+
+                }
+                System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
+            }
+        });
+        buffer.start();
+
+        try {
+            for (int i = 0; i < transactionSize * 10; i++) {
+                if (i % transactionSize == 0) {
+                    buffer.add(buildEntry("1", 1L + i, 40L + i, EntryType.TRANSACTIONBEGIN));
+                } else if ((i + 1) % transactionSize == 0) {
+                    buffer.add(buildEntry("1", 1L + i, 40L + i, EntryType.TRANSACTIONEND));
+                } else {
+                    buffer.add(buildEntry("1", 1L + i, 40L + i));
+                }
+            }
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        buffer.stop();
+    }
+
+    @Test
+    public void testForceFlush() {
+        final int bufferSize = 64;
+        EventTransactionBuffer buffer = new EventTransactionBuffer();
+        buffer.setBufferSize(bufferSize);
+        buffer.setFlushCallback(new TransactionFlushCallback() {
+
+            public void flush(List transaction) throws InterruptedException {
+                Assert.assertEquals(bufferSize, transaction.size());
+                System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+                for (Entry data : transaction) {
+
+                    CanalEntry.Header header = data.getHeader();
+                    Date date = new Date(header.getExecuteTime());
+                    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+                    if (data.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || data.getEntryType() == EntryType.TRANSACTIONEND) {
+                        // System.out.println(MessageFormat.format(messgae, new Object[] {
+                        // Thread.currentThread().getName(),
+                        // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                        // data.getEntry().getEntryType(), "" }));
+                        System.out.println(data.getEntryType());
+
+                    } else {
+                        System.out.println(MessageFormat.format(messgae, new Object[] {
+                                Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(),
+                                format.format(date), header.getSchemaName(), header.getTableName() }));
+                    }
+
+                }
+                System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
+            }
+        });
+        buffer.start();
+
+        try {
+            for (int i = 0; i < bufferSize * 2 + 1; i++) {
+                buffer.add(buildEntry("1", 1L + i, 40L + i));
+            }
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        buffer.stop();
+    }
+
+    private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        return entryBuilder.build();
+    }
+
+    private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        entryBuilder.setEntryType(type);
+        return entryBuilder.build();
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java
new file mode 100644
index 00000000..b0af1256
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/TableMetaCacheTest.java
@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.parse.inbound;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlConnection;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.TableMetaCache;
+
+public class TableMetaCacheTest {
+
+    @Test
+    public void testSimple() {
+
+        MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
+        try {
+            connection.connect();
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        TableMetaCache cache = new TableMetaCache(connection);
+        TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
+        Assert.assertNotNull(meta);
+        for (FieldMeta field : meta.getFileds()) {
+            System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : "
+                               + field.isNullable());
+        }
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java
new file mode 100644
index 00000000..79857912
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/DummyEventStore.java
@@ -0,0 +1,158 @@
+package com.alibaba.otter.canal.parse.inbound.group;
+
+import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+public class DummyEventStore implements CanalEventStore {
+
+    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+    private static final String messgae     = "{0} [{1}:{2}:{3}] {4} {5}.{6}";
+
+    public void ack(Position position) throws CanalStoreException {
+
+    }
+
+    public Events get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
+        return null;
+    }
+
+    public Events get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
+                                                                                 CanalStoreException {
+        return null;
+    }
+
+    public Position getFirstPosition() throws CanalStoreException {
+        return null;
+    }
+
+    public Position getLatestPosition() throws CanalStoreException {
+        return null;
+    }
+
+    public void rollback() throws CanalStoreException {
+
+    }
+
+    public Events tryGet(Position start, int batchSize) throws CanalStoreException {
+        return null;
+    }
+
+    public boolean isStart() {
+        return false;
+    }
+
+    public void start() {
+
+    }
+
+    public void stop() {
+
+    }
+
+    public void cleanAll() throws CanalStoreException {
+    }
+
+    public void cleanUntil(Position position) throws CanalStoreException {
+
+    }
+
+    public void put(Event data) throws InterruptedException, CanalStoreException {
+        put(Arrays.asList(data));
+    }
+
+    public boolean put(Event data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        return put(Arrays.asList(data), timeout, unit);
+    }
+
+    public boolean tryPut(Event data) throws CanalStoreException {
+        return tryPut(Arrays.asList(data));
+    }
+
+    public void put(List datas) throws InterruptedException, CanalStoreException {
+        for (Event data : datas) {
+            CanalEntry.Header header = data.getEntry().getHeader();
+            Date date = new Date(header.getExecuteTime());
+            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+            if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
+                || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
+                // System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
+                // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                // data.getEntry().getEntryType(), "" }));
+                System.out.println(data.getEntry().getEntryType());
+
+            } else {
+                System.out.println(MessageFormat.format(messgae,
+                                                        new Object[] { Thread.currentThread().getName(),
+                                                                header.getLogfileName(),
+                                                                String.valueOf(header.getLogfileOffset()),
+                                                                format.format(date), header.getEventType(),
+                                                                header.getSchemaName(), header.getTableName() }));
+            }
+        }
+    }
+
+    public boolean put(List datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        for (Event data : datas) {
+            CanalEntry.Header header = data.getEntry().getHeader();
+            Date date = new Date(header.getExecuteTime());
+            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+            if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
+                || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
+                // System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
+                // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                // data.getEntry().getEntryType(), "" }));
+                System.out.println(data.getEntry().getEntryType());
+
+            } else {
+                System.out.println(MessageFormat.format(messgae,
+                                                        new Object[] { Thread.currentThread().getName(),
+                                                                header.getLogfileName(),
+                                                                String.valueOf(header.getLogfileOffset()),
+                                                                format.format(date), header.getEventType(),
+                                                                header.getSchemaName(), header.getTableName() }));
+            }
+        }
+        return true;
+    }
+
+    public boolean tryPut(List datas) throws CanalStoreException {
+        System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+        for (Event data : datas) {
+
+            CanalEntry.Header header = data.getEntry().getHeader();
+            Date date = new Date(header.getExecuteTime());
+            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
+            if (data.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN
+                || data.getEntry().getEntryType() == EntryType.TRANSACTIONEND) {
+                // System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(),
+                // header.getLogfilename(), header.getLogfileoffset(), format.format(date),
+                // data.getEntry().getEntryType(), "" }));
+                System.out.println(data.getEntry().getEntryType());
+
+            } else {
+                System.out.println(MessageFormat.format(messgae,
+                                                        new Object[] { Thread.currentThread().getName(),
+                                                                header.getLogfileName(),
+                                                                String.valueOf(header.getLogfileOffset()),
+                                                                format.format(date), header.getEventType(),
+                                                                header.getSchemaName(), header.getTableName() }));
+            }
+
+        }
+        System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
+        return true;
+    }
+
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java
new file mode 100644
index 00000000..b632639c
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/group/GroupEventPaserTest.java
@@ -0,0 +1,103 @@
+package com.alibaba.otter.canal.parse.inbound.group;
+
+import java.net.InetSocketAddress;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.exception.CanalParseException;
+import com.alibaba.otter.canal.parse.inbound.AbstractBinlogParser;
+import com.alibaba.otter.canal.parse.inbound.BinlogParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.entry.EntryEventSink;
+import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+public class GroupEventPaserTest {
+
+    private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private static final String USERNAME      = "xxxxx";
+    private static final String PASSWORD      = "xxxxx";
+
+    @Test
+    public void testMysqlWithMysql() {
+        // MemoryEventStoreWithBuffer eventStore = new
+        // MemoryEventStoreWithBuffer();
+        // eventStore.setBufferSize(8196);
+
+        GroupEventSink eventSink = new GroupEventSink(3);
+        eventSink.setFilterTransactionEntry(false);
+        eventSink.setEventStore(new DummyEventStore());
+        eventSink.start();
+
+        // 构造第一个mysql
+        MysqlEventParser mysqlEventPaser1 = buildEventParser(3344);
+        mysqlEventPaser1.setEventSink(eventSink);
+        // 构造第二个mysql
+        MysqlEventParser mysqlEventPaser2 = buildEventParser(3345);
+        mysqlEventPaser2.setEventSink(eventSink);
+        // 构造第二个mysql
+        MysqlEventParser mysqlEventPaser3 = buildEventParser(3346);
+        mysqlEventPaser3.setEventSink(eventSink);
+        // 启动
+        mysqlEventPaser1.start();
+        mysqlEventPaser2.start();
+        mysqlEventPaser3.start();
+
+        try {
+            Thread.sleep(30 * 10 * 1000L);
+        } catch (InterruptedException e) {
+        }
+
+        mysqlEventPaser1.stop();
+        mysqlEventPaser2.stop();
+        mysqlEventPaser3.stop();
+    }
+
+    private MysqlEventParser buildEventParser(int slaveId) {
+        MysqlEventParser mysqlEventPaser = new MysqlEventParser();
+        EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+        mysqlEventPaser.setDestination("group-" + slaveId);
+        mysqlEventPaser.setSlaveId(slaveId);
+        mysqlEventPaser.setDetectingEnable(false);
+        mysqlEventPaser.setDetectingSQL(DETECTING_SQL);
+        mysqlEventPaser.setMasterInfo(buildAuthentication());
+        mysqlEventPaser.setMasterPosition(defaultPosition);
+        mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
+        mysqlEventPaser.setEventSink(new EntryEventSink());
+        mysqlEventPaser.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                // System.out.println(logPosition);
+            }
+
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+        return mysqlEventPaser;
+    }
+
+    private BinlogParser buildParser(AuthenticationInfo info) {
+        return new AbstractBinlogParser() {
+
+            public Entry parse(LogEvent event) throws CanalParseException {
+                // return _parser.parse(event);
+                return null;
+            }
+        };
+    }
+
+    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+        return new EntryPosition(binlogFile, offest, timestamp);
+    }
+
+    private AuthenticationInfo buildAuthentication() {
+        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java
new file mode 100644
index 00000000..e37077ea
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogDumpTest.java
@@ -0,0 +1,111 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class LocalBinlogDumpTest {
+
+    @Test
+    public void testSimple() {
+        String directory = "/home/jianghang/tmp/binlog";
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        final EntryPosition startPosition = new EntryPosition("mysql-bin.000006", 4L);
+
+        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
+        controller.setConnectionCharset(Charset.forName("UTF-8"));
+        controller.setDirectory(directory);
+        controller.setMasterPosition(startPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException,
+                                                                                                        InterruptedException {
+
+                for (Entry entry : entrys) {
+                    if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || entry.getEntryType() == EntryType.TRANSACTIONEND) {
+                        continue;
+                    }
+
+                    if (entry.getEntryType() == EntryType.ROWDATA) {
+                        RowChange rowChage = null;
+                        try {
+                            rowChage = RowChange.parseFrom(entry.getStoreValue());
+                        } catch (Exception e) {
+                            throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+                                                       + entry.toString(), e);
+                        }
+
+                        EventType eventType = rowChage.getEventType();
+                        System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
+                            entry.getHeader().getLogfileName(),
+                            entry.getHeader().getLogfileOffset(),
+                            entry.getHeader().getSchemaName(),
+                            entry.getHeader().getTableName(),
+                            eventType));
+
+                        for (RowData rowData : rowChage.getRowDatasList()) {
+                            if (eventType == EventType.DELETE) {
+                                print(rowData.getBeforeColumnsList());
+                            } else if (eventType == EventType.INSERT) {
+                                print(rowData.getAfterColumnsList());
+                            } else {
+                                System.out.println("-------> before");
+                                print(rowData.getBeforeColumnsList());
+                                System.out.println("-------> after");
+                                print(rowData.getAfterColumnsList());
+                            }
+                        }
+                    }
+                }
+
+                return true;
+            }
+
+        });
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        try {
+            Thread.sleep(100 * 1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        controller.stop();
+    }
+
+    private void print(List columns) {
+        for (Column column : columns) {
+            System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
+        }
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java
new file mode 100644
index 00000000..af31caec
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/LocalBinlogEventParserTest.java
@@ -0,0 +1,238 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class LocalBinlogEventParserTest {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private static final String USERNAME      = "xxxxx";
+    private static final String PASSWORD      = "xxxxx";
+    private String              directory;
+
+    @Before
+    public void setUp() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
+        File dummyFile = new File(url.getFile());
+        directory = new File(dummyFile.getParent() + "/binlog").getPath();
+    }
+
+    @Test
+    public void test_position() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker();
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setDirectory(directory);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                entryCount.incrementAndGet();
+
+                for (Entry entry : entrys) {
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        assertEquals(entryPosition, defaultPosition);
+    }
+
+    @Test
+    public void test_timestamp() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(300 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, 1322803601000L);
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setDirectory(directory);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            @Override
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+        assertTrue(entryPosition.getPosition() <= 6163L);
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    @Test
+    public void test_no_position() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000);
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000002",
+            null,
+            new Date().getTime() + 1000 * 1000L);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final LocalBinlogEventParser controller = new LocalBinlogEventParser();
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setDirectory(directory);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        // assertEquals(entryPosition.getJournalName(), "mysql-bin.000002");
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+        return new EntryPosition(binlogFile, offest, timestamp);
+    }
+
+    private AuthenticationInfo buildAuthentication() {
+        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
new file mode 100644
index 00000000..228dc851
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlDumpTest.java
@@ -0,0 +1,114 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Column;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
+import com.alibaba.otter.canal.protocol.CanalEntry.RowData;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class MysqlDumpTest {
+
+    @Test
+    public void testSimple() {
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L);
+
+        controller.setConnectionCharset(Charset.forName("UTF-8"));
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(false);
+        controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
+        controller.setMasterPosition(startPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException,
+                                                                                                        InterruptedException {
+
+                for (Entry entry : entrys) {
+                    if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN
+                        || entry.getEntryType() == EntryType.TRANSACTIONEND
+                        || entry.getEntryType() == EntryType.HEARTBEAT) {
+                        continue;
+                    }
+
+                    RowChange rowChage = null;
+                    try {
+                        rowChage = RowChange.parseFrom(entry.getStoreValue());
+                    } catch (Exception e) {
+                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:"
+                                                   + entry.toString(), e);
+                    }
+
+                    EventType eventType = rowChage.getEventType();
+                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s",
+                        entry.getHeader().getLogfileName(),
+                        entry.getHeader().getLogfileOffset(),
+                        entry.getHeader().getSchemaName(),
+                        entry.getHeader().getTableName(),
+                        eventType));
+
+                    if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
+                        System.out.println(" sql ----> " + rowChage.getSql());
+                    }
+
+                    for (RowData rowData : rowChage.getRowDatasList()) {
+                        if (eventType == EventType.DELETE) {
+                            print(rowData.getBeforeColumnsList());
+                        } else if (eventType == EventType.INSERT) {
+                            print(rowData.getAfterColumnsList());
+                        } else {
+                            System.out.println("-------> before");
+                            print(rowData.getBeforeColumnsList());
+                            System.out.println("-------> after");
+                            print(rowData.getAfterColumnsList());
+                        }
+                    }
+                }
+
+                return true;
+            }
+
+        });
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        try {
+            Thread.sleep(100 * 1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        controller.stop();
+    }
+
+    private void print(List columns) {
+        for (Column column : columns) {
+            System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
+        }
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java
new file mode 100644
index 00000000..32aa2f94
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParserTest.java
@@ -0,0 +1,308 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.helper.TimeoutChecker;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest;
+import com.alibaba.otter.canal.parse.stub.AbstractCanalLogPositionManager;
+import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+public class MysqlEventParserTest {
+
+    private static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private static final String USERNAME      = "xxxxx";
+    private static final String PASSWORD      = "xxxxx";
+
+    @Test
+    public void test_position() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker();
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(true);
+        controller.setDetectingSQL(DETECTING_SQL);
+        controller.setMasterPosition(defaultPosition);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            @Override
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        Assert.assertEquals(entryPosition, defaultPosition);
+    }
+
+    @Test
+    public void test_timestamp() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition(null, null, 1322803601000L);
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(true);
+        controller.setDetectingSQL(DETECTING_SQL);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setMasterPosition(defaultPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            @Override
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+        assertTrue(entryPosition.getPosition() <= 6163L);
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    @Test
+    public void test_ha() throws InterruptedException {
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(false);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setMasterPosition(defaultPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            @Override
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    String logfilename = entry.getHeader().getLogfileName();
+                    long logfileoffset = entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    entryPosition.setJournalName(logfilename);
+                    entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            public LogPosition getLatestIndexBy(String destination) {
+                LogPosition masterLogPosition = new LogPosition();
+                masterLogPosition.setIdentity(new LogIdentity(new InetSocketAddress("127.0.0.1", 3306), 1234L));
+                masterLogPosition.setPostion(new EntryPosition(1322803601000L));
+                return masterLogPosition;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
+        assertTrue(entryPosition.getPosition() <= 6163L);
+        assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
+    }
+
+    @Test
+    public void test_no_position() throws InterruptedException { // 在某个文件下，找不到对应的timestamp数据，会使用106L
+                                                                 // position进行数据抓取
+        final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 60 * 1000);
+        final AtomicLong entryCount = new AtomicLong(0);
+        final EntryPosition entryPosition = new EntryPosition();
+
+        final MysqlEventParser controller = new MysqlEventParser();
+        final EntryPosition defaultPosition = buildPosition("mysql-bin.000001",
+            null,
+            new Date().getTime() + 1000 * 1000L);
+        controller.setSlaveId(3344L);
+        controller.setDetectingEnable(false);
+        controller.setMasterInfo(buildAuthentication());
+        controller.setMasterPosition(defaultPosition);
+        controller.setEventSink(new AbstractCanalEventSinkTest>() {
+
+            @Override
+            public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                        throws CanalSinkException {
+                for (Entry entry : entrys) {
+                    entryCount.incrementAndGet();
+
+                    // String logfilename = entry.getHeader().getLogfileName();
+                    // long logfileoffset =
+                    // entry.getHeader().getLogfileOffset();
+                    long executeTime = entry.getHeader().getExecuteTime();
+
+                    // entryPosition.setJournalName(logfilename);
+                    // entryPosition.setPosition(logfileoffset);
+                    entryPosition.setTimestamp(executeTime);
+                    break;
+                }
+
+                controller.stop();
+                timeoutChecker.stop();
+                timeoutChecker.touch();
+                return true;
+            }
+        });
+
+        controller.setLogPositionManager(new AbstractCanalLogPositionManager() {
+
+            public void persistLogPosition(String destination, LogPosition logPosition) {
+                System.out.println(logPosition);
+            }
+
+            @Override
+            public LogPosition getLatestIndexBy(String destination) {
+                return null;
+            }
+        });
+
+        controller.start();
+        timeoutChecker.waitForIdle();
+
+        if (controller.isStart()) {
+            controller.stop();
+        }
+
+        // check
+        assertTrue(entryCount.get() > 0);
+
+        // 对比第一条数据和起始的position相同
+        // Assert.assertEquals(logfilename, "mysql-bin.000001");
+        // Assert.assertEquals(106L, logfileoffset);
+        assertTrue(entryPosition.getTimestamp() < defaultPosition.getTimestamp());
+    }
+
+    // ======================== helper method =======================
+
+    private EntryPosition buildPosition(String binlogFile, Long offest, Long timestamp) {
+        return new EntryPosition(binlogFile, offest, timestamp);
+    }
+
+    private AuthenticationInfo buildAuthentication() {
+        return new AuthenticationInfo(new InetSocketAddress(MYSQL_ADDRESS, 3306), USERNAME, PASSWORD);
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java
new file mode 100644
index 00000000..129e191b
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/SimpleDdlParserTest.java
@@ -0,0 +1,170 @@
+package com.alibaba.otter.canal.parse.inbound.mysql;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser;
+import com.alibaba.otter.canal.parse.inbound.mysql.dbsync.SimpleDdlParser.DdlResult;
+
+public class SimpleDdlParserTest {
+
+    @Test
+    public void testCreate() {
+        String queryString = "CREATE TABLE retl_mark ( `ID` int(11)";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE IF NOT EXISTS retl.retl_mark ( `ID` int(11)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE IF NOT EXISTS `retl_mark` ( `ID` int(11)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE IF NOT EXISTS `retl.retl_mark` ( `ID` int(11)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE  `retl`.`retl_mark` (\n  `ID` int(10) unsigned NOT NULL";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE TABLE  `retl`.`retl_mark`(\n  `ID` int(10) unsigned NOT NULL";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "CREATE table `bak591`.`j_order_log_back_201309` like j_order_log";
+        result = SimpleDdlParser.parse(queryString, "bak");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("bak591", result.getSchemaName());
+        Assert.assertEquals("j_order_log_back_201309", result.getTableName());
+
+        queryString = "CREATE TABLE `bak591`.`cm_settle_incash` (    `batch_id` bigint(20) NOT NULL  DEFAULT '0.00'";
+        result = SimpleDdlParser.parse(queryString, "bak");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("bak591", result.getSchemaName());
+        Assert.assertEquals("cm_settle_incash", result.getTableName());
+    }
+
+    @Test
+    public void testDrop() {
+        String queryString = "DROP TABLE retl_mark";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "DROP TABLE IF EXISTS retl.retl_mark;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "DROP TABLE IF EXISTS \n `retl.retl_mark` /;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "DROP /*!40005 TEMPORARY */  /*!40005 TEMPORARY */ TABLE IF EXISTS `temp_bond_keys`.`temp_bond_key_id`;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("temp_bond_keys", result.getSchemaName());
+        Assert.assertEquals("temp_bond_key_id", result.getTableName());
+    }
+
+    @Test
+    public void testAlert() {
+        String queryString = "alter table retl_mark drop index emp_name";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "alter table retl.retl_mark drop index emp_name";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "alter table \n `retl.retl_mark` drop index emp_name;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+    }
+
+    @Test
+    public void testTruncate() {
+        String queryString = "truncate table retl_mark";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "truncate table retl.retl_mark";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "truncate \n  `retl.retl_mark` ";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl_mark", result.getTableName());
+    }
+
+    @Test
+    public void testRename() {
+        String queryString = "rename table retl_mark to retl_mark2";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getOriSchemaName());
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getOriTableName());
+        Assert.assertEquals("retl_mark2", result.getTableName());
+
+        queryString = "rename table retl.retl_mark to retl2.retl_mark2";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getOriSchemaName());
+        Assert.assertEquals("retl2", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getOriTableName());
+        Assert.assertEquals("retl_mark2", result.getTableName());
+
+        queryString = "rename \n table \n `retl`.`retl_mark` to `retl2.retl_mark2`;";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getOriSchemaName());
+        Assert.assertEquals("retl2", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getOriTableName());
+        Assert.assertEquals("retl_mark2", result.getTableName());
+    }
+
+    @Test
+    public void testIndex() {
+        String queryString = "CREATE UNIQUE INDEX index_1 ON retl_mark(id,x)";
+        DdlResult result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+
+        queryString = "create index idx_qca_cid_mcid on q_contract_account (contract_id,main_contract_id)";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("q_contract_account", result.getTableName());
+
+        queryString = "DROP INDEX index_str ON retl_mark";
+        result = SimpleDdlParser.parse(queryString, "retl");
+        Assert.assertNotNull(result);
+        Assert.assertEquals("retl", result.getSchemaName());
+        Assert.assertEquals("retl_mark", result.getTableName());
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManagerTest.java
new file mode 100644
index 00000000..202ca241
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractLogPositionManagerTest.java
@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+
+import junit.framework.Assert;
+
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class AbstractLogPositionManagerTest extends AbstractZkTest {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+
+    public LogPosition doTest(CanalLogPositionManager logPositionManager) {
+        LogPosition getPosition = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertNull(getPosition);
+
+        LogPosition postion1 = buildPosition(1);
+        logPositionManager.persistLogPosition(destination, postion1);
+        LogPosition getPosition1 = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertEquals(postion1, getPosition1);
+
+        LogPosition postion2 = buildPosition(2);
+        logPositionManager.persistLogPosition(destination, postion2);
+        LogPosition getPosition2 = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertEquals(postion2, getPosition2);
+        return postion2;
+    }
+
+    protected LogPosition buildPosition(int number) {
+        LogPosition position = new LogPosition();
+        position.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        position.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
+        return position;
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java
new file mode 100644
index 00000000..ce3eadc0
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/AbstractZkTest.java
@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.junit.Assert;
+
+public class AbstractZkTest {
+
+    protected String destination = "ljhtest1";
+    protected String cluster1    = "127.0.0.1:2188";
+    protected String cluster2    = "127.0.0.1:2188,127.0.0.1:2188";
+
+    public void sleep(long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java
new file mode 100644
index 00000000..15e01a34
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManagerTest.java
@@ -0,0 +1,49 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.io.File;
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class FileMixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private static final String tmp     = System.getProperty("java.io.tmpdir", "/tmp");
+    private static final File   dataDir = new File(tmp, "canal");
+
+    @Before
+    public void setUp() {
+        try {
+            FileUtils.deleteDirectory(dataDir);
+        } catch (IOException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+
+    @Test
+    public void testAll() {
+        FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager();
+        logPositionManager.setDataDir(dataDir);
+        logPositionManager.setPeriod(100);
+        logPositionManager.start();
+
+        LogPosition position2 = doTest(logPositionManager);
+        sleep(1500);
+
+        FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager();
+        logPositionManager2.setDataDir(dataDir);
+        logPositionManager2.setPeriod(100);
+        logPositionManager2.start();
+
+        LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
+        Assert.assertEquals(position2, getPosition2);
+
+        logPositionManager.stop();
+        logPositionManager2.stop();
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java
new file mode 100644
index 00000000..88850443
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/MemoryLogPositionManagerTest.java
@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.junit.Test;
+
+public class MemoryLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    @Test
+    public void testAll() {
+        MemoryLogPositionManager logPositionManager = new MemoryLogPositionManager();
+        logPositionManager.start();
+        doTest(logPositionManager);
+        logPositionManager.stop();
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java
new file mode 100644
index 00000000..255d23f9
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/MetaLogPositionManagerTest.java
@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.parse.index;
+
+import java.net.InetSocketAddress;
+import java.util.Date;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.meta.MixedMetaManager;
+import com.alibaba.otter.canal.meta.ZooKeeperMetaManager;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+public class MetaLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+    private ZkClientx           zkclientx     = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        MixedMetaManager metaManager = new MixedMetaManager();
+
+        ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
+        zooKeeperMetaManager.setZkClientx(zkclientx);
+
+        metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
+        metaManager.start();
+
+        MetaLogPositionManager logPositionManager = new MetaLogPositionManager();
+        logPositionManager.setMetaManager(metaManager);
+        logPositionManager.start();
+        // 构建meta信息
+        ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
+        metaManager.subscribe(client1);
+
+        PositionRange range1 = buildRange(1);
+        metaManager.updateCursor(client1, range1.getEnd());
+
+        PositionRange range2 = buildRange(2);
+        metaManager.updateCursor(client1, range2.getEnd());
+
+        ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
+        metaManager.subscribe(client2);
+
+        PositionRange range3 = buildRange(3);
+        metaManager.updateCursor(client2, range3.getEnd());
+
+        PositionRange range4 = buildRange(4);
+        metaManager.updateCursor(client2, range4.getEnd());
+
+        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
+        Assert.assertEquals(range2.getEnd(), logPosition);
+
+        metaManager.stop();
+        logPositionManager.stop();
+    }
+
+    private PositionRange buildRange(int number) {
+        LogPosition start = new LogPosition();
+        start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
+
+        LogPosition end = new LogPosition();
+        end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
+        end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
+        return new PositionRange(start, end);
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java
new file mode 100644
index 00000000..99e78081
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManagerTest.java
@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.parse.index;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class MixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        MixedLogPositionManager logPositionManager = new MixedLogPositionManager();
+
+        ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
+        zookeeperLogPositionManager.setZkClientx(zkclientx);
+
+        logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager.start();
+
+        LogPosition position2 = doTest(logPositionManager);
+        sleep(1000);
+
+        MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager();
+        logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager2.start();
+
+        LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
+        Assert.assertEquals(position2, getPosition2);
+
+        logPositionManager.stop();
+        logPositionManager2.stop();
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java
new file mode 100644
index 00000000..49db15e1
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManagerTest.java
@@ -0,0 +1,52 @@
+package com.alibaba.otter.canal.parse.index;
+
+import junit.framework.Assert;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+
+public class PeriodMixedLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        PeriodMixedLogPositionManager logPositionManager = new PeriodMixedLogPositionManager();
+
+        ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager();
+        zookeeperLogPositionManager.setZkClientx(zkclientx);
+
+        logPositionManager.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager.start();
+
+        LogPosition position2 = doTest(logPositionManager);
+        sleep(1500);
+
+        PeriodMixedLogPositionManager logPositionManager2 = new PeriodMixedLogPositionManager();
+        logPositionManager2.setZooKeeperLogPositionManager(zookeeperLogPositionManager);
+        logPositionManager2.start();
+
+        LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
+        Assert.assertEquals(position2, getPosition2);
+
+        logPositionManager.stop();
+        logPositionManager2.stop();
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java
new file mode 100644
index 00000000..78f25339
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/index/ZooKeeperLogPositionManagerTest.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.parse.index;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+
+public class ZooKeeperLogPositionManagerTest extends AbstractLogPositionManagerTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testAll() {
+        ZooKeeperLogPositionManager logPositionManager = new ZooKeeperLogPositionManager();
+        logPositionManager.setZkClientx(zkclientx);
+        logPositionManager.start();
+
+        doTest(logPositionManager);
+        logPositionManager.stop();
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java b/parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java
new file mode 100644
index 00000000..081c4764
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalEventSinkTest.java
@@ -0,0 +1,11 @@
+package com.alibaba.otter.canal.parse.stub;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+
+public abstract class AbstractCanalEventSinkTest extends AbstractCanalLifeCycle implements CanalEventSink {
+
+    public void interrupt() {
+        // do nothing
+    }
+}
diff --git a/parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java b/parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java
new file mode 100644
index 00000000..df73735a
--- /dev/null
+++ b/parse/src/test/java/com/alibaba/otter/canal/parse/stub/AbstractCanalLogPositionManager.java
@@ -0,0 +1,8 @@
+package com.alibaba.otter.canal.parse.stub;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.parse.index.CanalLogPositionManager;
+
+public abstract class AbstractCanalLogPositionManager extends AbstractCanalLifeCycle implements CanalLogPositionManager {
+
+}
diff --git a/parse/src/test/resources/binlog/mysql-bin.000001 b/parse/src/test/resources/binlog/mysql-bin.000001
new file mode 100644
index 0000000000000000000000000000000000000000..943ae0a060330ca87ced4fc9c9dcf7138c4ff251
GIT binary patch
literal 22920
zcmc(n3v?Xib;m!;l28rCvYcYavGCZbEI(ySmI-#S(MnpGwb!z;S{W&^kc7381$t$*
z_A$?bP96|yC<40%M=2pu`Z%YBgg!Vx>50KWfRkbs5=aqAU6dvV4!DF8mr&IEzw_tg+hSbQ+vGZb6bJ=nXhqw`=#|Grq~y550q&Q1P;%z{hjssdG{a_Of?sYaE%xPamX
z1qC@%$6u+>!BK)V9^#@6cksVdo>GhHzp9i%oPY9X4<$Nt3rMU$#o_})9jZFGJyZ{o
z1xVUR<1}X){9CC!Gp!&I8%osm^u|=q?0Gpy6(4sYN-n3djQ*ognI*F*_7MV0YPam|
zqKbHIXrT9CEWS4>QFhH>H;!0{tk2N64F92S_$7WTm48w@!k?GtK>6jv9LerznOY`}
zY9hfvG-yNv)#0FFv@?HuS$p#6ZM{AH2M)CxE9X$Kr?cJY=})XIFJEQU?~WSvP2uob
z1OOh<;O)l2j`+U)9r2Z$HmovggLQ$X@N{ZJB(y6KX)$&NTa1+uTs3Ev5v-@K2DgU#
z`v+>PrKIMHK%_Ak-FhI=b?JAWwai;Lxm4;MFD`lBAj0#unaUqvk!5D5lJj<<-F9;J
zP{^}aN$D?hkIh1vz4_}CeS_aaSV*e@KWqk|>1I$PW}e`K)xOO|>eZJmW@hE$#p^wr
zfryyd!Xh&>gTh6#=$K&|U!y{+nW>L5yGeh!0y73JG5KwV_-}18dj3z%)5csn*5A@3
z6{^$s!7ABvtwr4tb=o%H`wogFW!@EMv(0PhWsn_D=*F9
z@oiLIo)3|A_(ebZ99oFIXzh-gJ$v4~+@tEK%11S?VN?%Lb)l}FrDji^#dK6d{9;>b#U$Lj50IP{KZZx9i0bI*gW2XClm
zZ=0yDnyPKy_(YPvK~W@@TKSN@+S1d7dhKGm9^}x9`@uBBwae255wY|Oua;2FyR@DT
zT1M~1R!dW7vv|tzI`31ciPsR`+0~~$#&4zS_Pjd%_I@XaD!*rjX0%V)D@4TWi!9<{
z`cN%i7yp5~SN4>_E5D0CO}uU*HKii_XQeK<(&F`c4qfKiD@4TWqb%aVE2_n7{v3C&
z>?wm+?g7-q>oQWKYeDVxbc@An7l&FrdxeO2{Vt1m@QP~j`rZt@I?gN9#OvJi;q|!o
z`sNocULzdp^z0QP;`L8i#DiBxwT~yl%SyZRqyw6(ZvG
zMi%kl71iSPd8&5v>Nu}Z6R(rZa#DN!xz+3Hx$qkI>=h#7bw7)E@QP~j`VUm?=GAe(
zLQTA$VwN-7>x+9W^Lo*R@OqPHuMiQhQ5Nyw71iSPL8^B1>Nu}Z6R-coEQ{vjIzy><
zTPvxKzQ!ZwT7pK<8VJbQ(RcwIE#qgPam*Dk7d^XfRS
zP!q4WGE1BGdfe*u84itl_6ia4iZ5O~%`2+KYlN!ZygJS+)WmBavwT^5J!SR!0}egl
z*(*fE>lv?JQ7vA#EpYehIImC>uPvme)Q#HfZ>(Ow#i0j1dxeO2eVs)->{nEa*VUhL
z_v$#WP!q2~Qln4(+Uvr0%YOX^hsHd6g@|~4jzv6pMYVXn=nvh!I?gN9#A`XJDRsB@
zdV|&LK@L6S*(*fE>))}62d}7RuU8RQ(M73V-9Ce#wI$x2k)q6?|X^sf`v}r)2(H9&Z`01;=Gi!mHM%Ee$wi^mqU+ub`BA7
z{?{zx!8xkgc|CERO>>_9N#1ccLruIcSOl-HX|H#8Sax$Who1246(ZvGD2sUTifZxt
zCRMw=BRTG7sEOApW_d?@ebDN)_S0y?lb*doM7$2OhzGBz7OyW+wVPMRd4-yIeT!Mn
zYOim1S>|;EZn&s3p1nduy!Nw*2d}6WuTN05n^(tqg_?N%DYMiR=}!h$uS;<2M!oOZ
z>$_I3Ut|#vUQsPx|B|ZRygJS+)Wqvk%rdOK&hNI&YrztD{lK$Vh{(Lwv4{t+s1~pH
zQni~`$9aXCczuXj{zQ9SYW4a)hkonXD@4R=MKK!Y!7Hl8>n&96=GAdtp(b9BGs_X}
z^`c(Oy#9hiYyJYTUcNnshF@TPW=o1Jh>~%~H8jXE2Wiz1QqVU3uQVS8m-wpW5=d
zJ>{sBHF2COm1_7ebK=)pZT*TvBff1xOl%b|!*Segp-609NkukWxZ+CMs;IQtsz9aK
ziu2)qsBN7*tY-^1h5nI4U$|lxfM#|>lW&wjMC`oHBJOrj#Jjnfj#W$R!fI#gjFTUG
z@R;2!bj55N=_)nv1JjYs>~}eIj%HtD5szleKbvY6c*)62U6W^UmpvKWn6hAPX2YaQ
z%Gzx79?Q($&7s>o&n!e_W}jgZce5yxnZ5XrQq4N~aWY#8U76XrdA6OVqTjc!*K$;f
zt)KAWey43UT5WxHC2T$G+ZM#cRx^va+d`4p8lfVaE!(pFlFe2HD#g~{@ZoZPYrdQO
zYtmMb486ypQ`5Ha0kA5|Y*{XFAR=~3S5eUnt^`mdC)huxBCDO0E1Z;RvY%j6g3uMS
z_cP2T+UzSyv$(4JPY#VwoAtOfAtt7mte%-^6p85s71>SO-c?e3*-fwefomau&$&aR*}3u1);W*!|>&B5BvNROHYuyRQm+yPALG
z&@Ovr%C(EVvJynnuG4JeIo+<8{@l{8^<`KP3w*Z=V$!Zo7ISYGilkj1P?1Bs?7mXk
z_3#T0?Xp*ZRqF^u^ON>+!q+F`3~$7IQa^BH0L2RODbf#h3lO5qr^LBiJib
zHiEsf5=62#7FVDrd1uXYX^XX8-5grtyIl~Ic72n@+}njBZr8npwPB-UyX?MF*2wpD
zyUx>JLfN;Ay)>m?sik0&hV3LKx&x~lcH!WMyZHf^PI~bn#{J46vzPQj<}-)+fQ#B5
z3Dif8Ksbyq2ls@+!R^6DV|Tr=x@@%(-EEY{2L=+QRi!1RMag6AB*_mw9iyxFz|ek+
zerr(^K
ziKMFNYs;A%>88KcpD;RldkuQN=^#BxL(h;H1O43h#DJymMo%Yw58Tt$6YDgtzRnD=
zkFO^W4j*@TI(Vv&rnp`|jU@F3NfpS1(7E4f{lsI5UZuYRuMN=iM1jVjffq%xog}wi
z;-xiOuBkB?VX4%qgnX&!A0z#rc+xkQd}A;itck8QBEg1ms3s5%(V}k*2lfQ*FV%!Y
zw2o{$0*A+raON5&uIA5q@onb{)suhXYigjr5>zem;@`pMU`-P}yG6L|=tO&Rqb9MW
zGlN9%oDYJg#=v&gmvcZ;+THPvOaDI|C&=-WpEyCr$(Js-(lI}?+3I>GTc$R$yFovG
z7pkK<4K+p^d9kbU9z=v5Z<-B)_enUEdgu1c;YT0?}|=byHnk
zFako`eX>yPDuW(ZoWb*o($~jn+75sI-1~~P;M`Q|OMQJXoVl&32I(2homuP&`c;Jf
zmS^;g%Nb2AYp!hyg!d%hU*0G6pI>%PjXBhLpre;>u+l`OK9_2wE;_?#HgNZp9t-Om
zl*J*pP-~hZ5n7yWvOIA6GjFUWNZ1foaN!wvq{fk^1Nm%{ZjvYoq0`0Cy30~
z{N^=T^1A|TE%Vu9Xz?{HK3+Go7VeSaCzrGGxGVJJ>Us1Rv?zHY4ur3pZ>HozWhTpW
ztH{8RtI7LVa@l5l_ED;TNM8+reDd7Lp(W%++aRBv9LapujSKW@O16@l4@?7_pPWW6
zhInXY@fF&`hZN!~;wwmMj6*Nz?$ZmIyKlcSZFU|y!awf9k7VJ`p@+WH`GL&*$i)!;
zVmu_S_G$i)6Bhm$hkopbA3DN6!aCXek7VILk&z$B%#U0Q;h$jfxaOatko3Q#3f6q_
zLr3@{tdpG|$-+NO$?5$EGV>!BL-KKRQ1p|XAIZXBP08u}
zKxTgAVhDdLi$Au-+{3ddB>f-bkS~7d2>-YXKazz%r#cfqkeMI37{Xsn`SeWj=S}_`
z3JHIVL%#T-Bm5(*lfC~)7XA~IoZf#RGd}~;AB2B`#g}f)fekgALc(8CgO_~qLr3@{
ztdpG|$-+NO$?5#SW`5*i2>&RHH*Pceb15YJlm7XkBm70R=tFjXBny8vC8zTPnfZ~6
zA^fc@e*5Jne;$Q|e~d%E`VSr9A9vwLvhe57x47y2KxTgAVhDdRzV%g)Rhj&EQAqe>
z96IIa{0|-BA7P#B{YSF!pUB7$WadXMhVW0Y_(;IyAEA)&m(;8viQB4
z{}_dYe~d%E`VSr9A9vwLvhe57l~+1HkeMI37{XtSD>ZdQ^B<>>@W(jhiyu0|Kf*fM
z`;TPdKar6i$jpyi4B?+(@uQmm?kk6kgxtj
zNBGBG_>nC9IrOz`IzNz^AGsL9UyQGD)pMHvNeT&nj6=Top(FewtdqU}NEZGR8To>v)g@nHZA0+*)f9ME*gmtp>BU$)|DLK9WKxTgAVhH~zi$AaVpQe!T
zPx|MFj_?=Z_MgN0N3!r&Q*t^#keMI37{cGm;xB0a?O(O-YXKazz%haMJ7=La(LBNs#Xiz#2JjveOp!ww2b|6?5T#Sb0fA7P#B>j%lg
ze
zzvgeCknm6X=ZB8)7d4;{+4+$y{MD44&JSefM=pl&x3c&*HUBV$gnx`f2mSOPI>JBh
z!jEL(&-p?oejqbHaxsLznDUkSwdVgSg@ixGAz%E^5&jX@$=-h?3;&6X{6J=Yk8#KsKXinD+=U;>!k-h(
z#1CZVM=pl&7gN4cKhylnDJ1+c4*BATj_{ALPWJVKWZ^%Nksrv+k6aAlpJ4HS)BGhA
z68@4VSo6gX9pR6#PIi7I3;!@Br>`F%Ge2@MgnyL9U)TIADJ1-p9P-5v9pNw9gFa;E
zN3!r&Q*t^#keMI37{cGm;uD&G4TXe%j6=Top(Fg`F8oLq{v3LeAe|q`%#U0Q;V;IM
z2kH&Y-$x1ko
vGV>!BL-<=+yiM~r-AP}M;(C7#joUbsi-Xfo-*rP@?p;U!^X0M)_^J3mo1HN{

literal 0
HcmV?d00001

diff --git a/parse/src/test/resources/binlog/mysql-bin.000002 b/parse/src/test/resources/binlog/mysql-bin.000002
new file mode 100644
index 0000000000000000000000000000000000000000..98065e5c2041490c35b8be932984c37b2f9d8f11
GIT binary patch
literal 302
zcmY+7O^U)m5QSgbR(m5Ox)I&E@1V>GE@ePO@MjTSxQwPp9GV!CC>~&N>2bV}w-6bv
zif$};R8{x)>isV_;rw)+emDSrNWUZp5*@egetsoUTbnR7aTIS7qt}tuez5n$l?k-n
zu8DoGYPST13M5qp@F44o2&I&;lp;7~L^@}-zXR`}G&HI8IQdnDT#^P-(J_U5!CQE<
z{x~+N#dm)&obatGvzm0tES;B8W>o$V?0gH6D*_DH7;cf+RmKx`Anx}Y|0KujG#i*@
PmHD+na(TdNB)H)LBv3TJ

literal 0
HcmV?d00001

diff --git a/parse/src/test/resources/dummy.txt b/parse/src/test/resources/dummy.txt
new file mode 100644
index 00000000..f77ca914
--- /dev/null
+++ b/parse/src/test/resources/dummy.txt
@@ -0,0 +1 @@
+本文件仅仅为定位绝对路径使用
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 00000000..bdfa5a6c
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,407 @@
+
+	4.0.0
+	com.alibaba.otter
+	canal
+	pom
+	canal module for otter ${project.version}
+	1.0.19-SNAPSHOT
+	https://github.com/alibaba/canal
+	
+	    org.sonatype.oss
+	    oss-parent
+	    7
+	
+	
+		
+			agapple
+			http://agapple.iteye.com
+			jianghang115@gmail.com
+			8
+		
+		
+			zavakid
+			http://www.zavakid.com
+			zava.kid@gmail.com
+			8
+		
+		
+			in355hz
+			http://in355hz.iteye.com
+			in355hz@gmail.com
+			8
+		
+	
+
+	
+		
+			Apache License, Version 2.0
+			http://www.apache.org/licenses/LICENSE-2.0
+		
+	
+
+	
+		git@github.com:alibaba/canal.git
+		scm:git:git@github.com:alibaba/canal.git
+		scm:git:git@github.com:alibaba/canal.git
+	
+	
+	
+		
+			central
+			http://repo1.maven.org/maven2
+			
+				true
+			
+			
+				false
+			
+		
+		
+			java.net
+			http://download.java.net/maven/2/
+			
+				true
+			
+			
+				false
+			
+		
+		
+			alibaba
+			http://code.alibabatech.com/mvn/releases/
+			
+				true
+			
+			
+				false
+			
+		
+		
+			sonatype
+			sonatype
+			https://oss.sonatype.org/content/repositories/snapshots
+			
+				false
+			
+			
+				true
+			
+		
+		
+			sonatype-release
+			sonatype-release
+			https://oss.sonatype.org/service/local/repositories/releases/content
+			
+				false
+			
+			
+				true
+			
+		
+	
+	
+	
+		UTF-8
+		
+		true
+		true
+		
+		1.6
+		1.6
+		UTF-8
+	
+	
+	
+		common
+		meta
+		dbsync
+		filter	
+		driver
+		parse
+		sink
+		store
+		protocol
+		instance
+		server
+		client
+		deployer
+		example
+	
+	
+	
+		
+			
+				org.springframework
+				spring
+				2.5.6
+			
+            
+            
+				commons-lang
+				commons-lang
+				2.6
+			
+			
+				commons-io
+				commons-io
+				2.4
+			
+			
+				org.apache.zookeeper
+				zookeeper
+				3.4.5
+				
+				  
+					log4j
+					log4j
+				  
+				  
+					org.slf4j
+					slf4j-log4j12
+				  
+				  
+					org.slf4j
+					slf4j-api
+				  
+				  
+					jline
+					jline
+				  
+				
+			
+			
+				com.github.sgroschupf
+				zkclient
+				0.1
+			
+			
+				com.alibaba
+				fastjson
+				1.1.26
+			
+			
+				com.google.guava
+				guava
+				r09
+			
+			
+				com.googlecode.aviator
+				aviator
+				2.2.1
+			
+			
+				oro
+				oro
+				2.0.8
+			
+			
+			  org.jboss.netty
+			  netty
+			  3.2.5.Final
+			
+			
+				com.google.protobuf
+				protobuf-java
+				2.4.1
+			
+            
+            
+				ch.qos.logback
+				logback-core
+				1.0.6
+			
+			
+				ch.qos.logback
+				logback-classic
+				1.0.6
+			
+			
+				org.slf4j
+				jcl-over-slf4j
+				1.6.0
+			
+			
+				org.slf4j
+				slf4j-api
+				1.6.0
+			
+			
+            
+				junit
+				junit
+				4.5
+				test
+			
+			
+				mysql
+				mysql-connector-java
+				5.1.12
+				test
+			
+		
+	
+	
+	
+		
+			
+				org.jvnet.wagon-svn
+				wagon-svn
+				1.9
+			
+			
+				org.apache.maven.wagon
+				wagon-http-shared
+				1.0-beta-7
+			
+		
+		
+			
+				org.apache.maven.plugins
+				maven-source-plugin
+				
+					
+						attach-sources
+						
+							jar
+						
+					
+				
+			
+			
+				org.apache.maven.plugins
+				maven-compiler-plugin
+				
+					${java_source_version}
+					${java_target_version}
+					${file_encoding}
+				
+			
+			
+			   org.apache.maven.plugins  
+			   maven-eclipse-plugin  
+			   2.5.1  
+			     
+				    
+					   
+						.settings/org.eclipse.core.resources.prefs  
+						  
+						   =${file_encoding}${line.separator}]]>  
+						  
+					   
+				    
+			     
+			
+			
+				org.apache.maven.plugins
+				maven-surefire-plugin
+				2.5
+				
+				  
+					**/*Test.java
+				  
+				  
+					**/*NoRunTest.java
+				  
+				
+			
+			
+                org.apache.maven.plugins
+                maven-surefire-plugin
+                2.5
+            
+            
+            
+		
+		src/main/java  
+		src/test/java
+		
+			
+				src/main/resources
+				
+					**/*
+				
+				
+					**/.svn/
+				
+			
+		
+		
+			
+				src/test/resources
+				
+					**/*
+				
+				
+					**/.svn/
+				
+			
+		
+	
+	
+	
+		
+			sonatype-nexus-snapshots
+			Sonatype Nexus Snapshots
+			https://oss.sonatype.org/content/repositories/snapshots/
+		
+		
+			sonatype-nexus-staging
+			Nexus Release Repository
+			https://oss.sonatype.org/service/local/staging/deploy/maven2/
+		
+	
+
diff --git a/protocol/pom.xml b/protocol/pom.xml
new file mode 100644
index 00000000..a4c013e5
--- /dev/null
+++ b/protocol/pom.xml
@@ -0,0 +1,30 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.protocol
+	jar
+	canal protocol module for otter ${project.version}
+	http://b2b-doc.alibaba-inc.com/display/opentech/Otter
+
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.google.protobuf
+			protobuf-java
+		
+		
+			commons-lang
+			commons-lang
+		
+	
+
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
new file mode 100755
index 00000000..0942c0d7
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalEntry.java
@@ -0,0 +1,7734 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: EntryProtocol.proto
+
+package com.alibaba.otter.canal.protocol;
+
+public final class CanalEntry {
+  private CanalEntry() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public enum EntryType
+      implements com.google.protobuf.ProtocolMessageEnum {
+    TRANSACTIONBEGIN(0, 1),
+    ROWDATA(1, 2),
+    TRANSACTIONEND(2, 3),
+    HEARTBEAT(3, 4),
+    ;
+    
+    public static final int TRANSACTIONBEGIN_VALUE = 1;
+    public static final int ROWDATA_VALUE = 2;
+    public static final int TRANSACTIONEND_VALUE = 3;
+    public static final int HEARTBEAT_VALUE = 4;
+    
+    
+    public final int getNumber() { return value; }
+    
+    public static EntryType valueOf(int value) {
+      switch (value) {
+        case 1: return TRANSACTIONBEGIN;
+        case 2: return ROWDATA;
+        case 3: return TRANSACTIONEND;
+        case 4: return HEARTBEAT;
+        default: return null;
+      }
+    }
+    
+    public static com.google.protobuf.Internal.EnumLiteMap
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public EntryType findValueByNumber(int number) {
+              return EntryType.valueOf(number);
+            }
+          };
+    
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.getDescriptor().getEnumTypes().get(0);
+    }
+    
+    private static final EntryType[] VALUES = {
+      TRANSACTIONBEGIN, ROWDATA, TRANSACTIONEND, HEARTBEAT, 
+    };
+    
+    public static EntryType valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+    
+    private final int index;
+    private final int value;
+    
+    private EntryType(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+    
+    // @@protoc_insertion_point(enum_scope:com.alibaba.otter.canal.protocol.EntryType)
+  }
+  
+  public enum EventType
+      implements com.google.protobuf.ProtocolMessageEnum {
+    INSERT(0, 1),
+    UPDATE(1, 2),
+    DELETE(2, 3),
+    CREATE(3, 4),
+    ALTER(4, 5),
+    ERASE(5, 6),
+    QUERY(6, 7),
+    TRUNCATE(7, 8),
+    RENAME(8, 9),
+    CINDEX(9, 10),
+    DINDEX(10, 11),
+    ;
+    
+    public static final int INSERT_VALUE = 1;
+    public static final int UPDATE_VALUE = 2;
+    public static final int DELETE_VALUE = 3;
+    public static final int CREATE_VALUE = 4;
+    public static final int ALTER_VALUE = 5;
+    public static final int ERASE_VALUE = 6;
+    public static final int QUERY_VALUE = 7;
+    public static final int TRUNCATE_VALUE = 8;
+    public static final int RENAME_VALUE = 9;
+    public static final int CINDEX_VALUE = 10;
+    public static final int DINDEX_VALUE = 11;
+    
+    
+    public final int getNumber() { return value; }
+    
+    public static EventType valueOf(int value) {
+      switch (value) {
+        case 1: return INSERT;
+        case 2: return UPDATE;
+        case 3: return DELETE;
+        case 4: return CREATE;
+        case 5: return ALTER;
+        case 6: return ERASE;
+        case 7: return QUERY;
+        case 8: return TRUNCATE;
+        case 9: return RENAME;
+        case 10: return CINDEX;
+        case 11: return DINDEX;
+        default: return null;
+      }
+    }
+    
+    public static com.google.protobuf.Internal.EnumLiteMap
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public EventType findValueByNumber(int number) {
+              return EventType.valueOf(number);
+            }
+          };
+    
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.getDescriptor().getEnumTypes().get(1);
+    }
+    
+    private static final EventType[] VALUES = {
+      INSERT, UPDATE, DELETE, CREATE, ALTER, ERASE, QUERY, TRUNCATE, RENAME, CINDEX, DINDEX, 
+    };
+    
+    public static EventType valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+    
+    private final int index;
+    private final int value;
+    
+    private EventType(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+    
+    // @@protoc_insertion_point(enum_scope:com.alibaba.otter.canal.protocol.EventType)
+  }
+  
+  public enum Type
+      implements com.google.protobuf.ProtocolMessageEnum {
+    ORACLE(0, 1),
+    MYSQL(1, 2),
+    PGSQL(2, 3),
+    ;
+    
+    public static final int ORACLE_VALUE = 1;
+    public static final int MYSQL_VALUE = 2;
+    public static final int PGSQL_VALUE = 3;
+    
+    
+    public final int getNumber() { return value; }
+    
+    public static Type valueOf(int value) {
+      switch (value) {
+        case 1: return ORACLE;
+        case 2: return MYSQL;
+        case 3: return PGSQL;
+        default: return null;
+      }
+    }
+    
+    public static com.google.protobuf.Internal.EnumLiteMap
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public Type findValueByNumber(int number) {
+              return Type.valueOf(number);
+            }
+          };
+    
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.getDescriptor().getEnumTypes().get(2);
+    }
+    
+    private static final Type[] VALUES = {
+      ORACLE, MYSQL, PGSQL, 
+    };
+    
+    public static Type valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+    
+    private final int index;
+    private final int value;
+    
+    private Type(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+    
+    // @@protoc_insertion_point(enum_scope:com.alibaba.otter.canal.protocol.Type)
+  }
+  
+  public interface EntryOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional .com.alibaba.otter.canal.protocol.Header header = 1;
+    boolean hasHeader();
+    com.alibaba.otter.canal.protocol.CanalEntry.Header getHeader();
+    com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder getHeaderOrBuilder();
+    
+    // optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+    boolean hasEntryType();
+    com.alibaba.otter.canal.protocol.CanalEntry.EntryType getEntryType();
+    
+    // optional bytes storeValue = 3;
+    boolean hasStoreValue();
+    com.google.protobuf.ByteString getStoreValue();
+  }
+  public static final class Entry extends
+      com.google.protobuf.GeneratedMessage
+      implements EntryOrBuilder {
+    // Use Entry.newBuilder() to construct.
+    private Entry(Builder builder) {
+      super(builder);
+    }
+    private Entry(boolean noInit) {}
+    
+    private static final Entry defaultInstance;
+    public static Entry getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Entry getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional .com.alibaba.otter.canal.protocol.Header header = 1;
+    public static final int HEADER_FIELD_NUMBER = 1;
+    private com.alibaba.otter.canal.protocol.CanalEntry.Header header_;
+    public boolean hasHeader() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Header getHeader() {
+      return header_;
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder getHeaderOrBuilder() {
+      return header_;
+    }
+    
+    // optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+    public static final int ENTRYTYPE_FIELD_NUMBER = 2;
+    private com.alibaba.otter.canal.protocol.CanalEntry.EntryType entryType_;
+    public boolean hasEntryType() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.EntryType getEntryType() {
+      return entryType_;
+    }
+    
+    // optional bytes storeValue = 3;
+    public static final int STOREVALUE_FIELD_NUMBER = 3;
+    private com.google.protobuf.ByteString storeValue_;
+    public boolean hasStoreValue() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public com.google.protobuf.ByteString getStoreValue() {
+      return storeValue_;
+    }
+    
+    private void initFields() {
+      header_ = com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance();
+      entryType_ = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.ROWDATA;
+      storeValue_ = com.google.protobuf.ByteString.EMPTY;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeMessage(1, header_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeEnum(2, entryType_.getNumber());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, storeValue_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(1, header_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(2, entryType_.getNumber());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, storeValue_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Entry parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Entry prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.EntryOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Entry.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getHeaderFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        if (headerBuilder_ == null) {
+          header_ = com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance();
+        } else {
+          headerBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        entryType_ = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.ROWDATA;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        storeValue_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Entry.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Entry getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Entry.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Entry build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Entry result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.Entry buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.Entry result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Entry buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Entry result = new com.alibaba.otter.canal.protocol.CanalEntry.Entry(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        if (headerBuilder_ == null) {
+          result.header_ = header_;
+        } else {
+          result.header_ = headerBuilder_.build();
+        }
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.entryType_ = entryType_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.storeValue_ = storeValue_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Entry) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Entry)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Entry other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Entry.getDefaultInstance()) return this;
+        if (other.hasHeader()) {
+          mergeHeader(other.getHeader());
+        }
+        if (other.hasEntryType()) {
+          setEntryType(other.getEntryType());
+        }
+        if (other.hasStoreValue()) {
+          setStoreValue(other.getStoreValue());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Header.newBuilder();
+              if (hasHeader()) {
+                subBuilder.mergeFrom(getHeader());
+              }
+              input.readMessage(subBuilder, extensionRegistry);
+              setHeader(subBuilder.buildPartial());
+              break;
+            }
+            case 16: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalEntry.EntryType value = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(2, rawValue);
+              } else {
+                bitField0_ |= 0x00000002;
+                entryType_ = value;
+              }
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              storeValue_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional .com.alibaba.otter.canal.protocol.Header header = 1;
+      private com.alibaba.otter.canal.protocol.CanalEntry.Header header_ = com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance();
+      private com.google.protobuf.SingleFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Header, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder, com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder> headerBuilder_;
+      public boolean hasHeader() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Header getHeader() {
+        if (headerBuilder_ == null) {
+          return header_;
+        } else {
+          return headerBuilder_.getMessage();
+        }
+      }
+      public Builder setHeader(com.alibaba.otter.canal.protocol.CanalEntry.Header value) {
+        if (headerBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          header_ = value;
+          onChanged();
+        } else {
+          headerBuilder_.setMessage(value);
+        }
+        bitField0_ |= 0x00000001;
+        return this;
+      }
+      public Builder setHeader(
+          com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder builderForValue) {
+        if (headerBuilder_ == null) {
+          header_ = builderForValue.build();
+          onChanged();
+        } else {
+          headerBuilder_.setMessage(builderForValue.build());
+        }
+        bitField0_ |= 0x00000001;
+        return this;
+      }
+      public Builder mergeHeader(com.alibaba.otter.canal.protocol.CanalEntry.Header value) {
+        if (headerBuilder_ == null) {
+          if (((bitField0_ & 0x00000001) == 0x00000001) &&
+              header_ != com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance()) {
+            header_ =
+              com.alibaba.otter.canal.protocol.CanalEntry.Header.newBuilder(header_).mergeFrom(value).buildPartial();
+          } else {
+            header_ = value;
+          }
+          onChanged();
+        } else {
+          headerBuilder_.mergeFrom(value);
+        }
+        bitField0_ |= 0x00000001;
+        return this;
+      }
+      public Builder clearHeader() {
+        if (headerBuilder_ == null) {
+          header_ = com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance();
+          onChanged();
+        } else {
+          headerBuilder_.clear();
+        }
+        bitField0_ = (bitField0_ & ~0x00000001);
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder getHeaderBuilder() {
+        bitField0_ |= 0x00000001;
+        onChanged();
+        return getHeaderFieldBuilder().getBuilder();
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder getHeaderOrBuilder() {
+        if (headerBuilder_ != null) {
+          return headerBuilder_.getMessageOrBuilder();
+        } else {
+          return header_;
+        }
+      }
+      private com.google.protobuf.SingleFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Header, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder, com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder> 
+          getHeaderFieldBuilder() {
+        if (headerBuilder_ == null) {
+          headerBuilder_ = new com.google.protobuf.SingleFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Header, com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder, com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder>(
+                  header_,
+                  getParentForChildren(),
+                  isClean());
+          header_ = null;
+        }
+        return headerBuilder_;
+      }
+      
+      // optional .com.alibaba.otter.canal.protocol.EntryType entryType = 2 [default = ROWDATA];
+      private com.alibaba.otter.canal.protocol.CanalEntry.EntryType entryType_ = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.ROWDATA;
+      public boolean hasEntryType() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.EntryType getEntryType() {
+        return entryType_;
+      }
+      public Builder setEntryType(com.alibaba.otter.canal.protocol.CanalEntry.EntryType value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000002;
+        entryType_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearEntryType() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        entryType_ = com.alibaba.otter.canal.protocol.CanalEntry.EntryType.ROWDATA;
+        onChanged();
+        return this;
+      }
+      
+      // optional bytes storeValue = 3;
+      private com.google.protobuf.ByteString storeValue_ = com.google.protobuf.ByteString.EMPTY;
+      public boolean hasStoreValue() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public com.google.protobuf.ByteString getStoreValue() {
+        return storeValue_;
+      }
+      public Builder setStoreValue(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        storeValue_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearStoreValue() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        storeValue_ = getDefaultInstance().getStoreValue();
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Entry)
+    }
+    
+    static {
+      defaultInstance = new Entry(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Entry)
+  }
+  
+  public interface HeaderOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int32 version = 1 [default = 1];
+    boolean hasVersion();
+    int getVersion();
+    
+    // optional string logfileName = 2;
+    boolean hasLogfileName();
+    String getLogfileName();
+    
+    // optional int64 logfileOffset = 3;
+    boolean hasLogfileOffset();
+    long getLogfileOffset();
+    
+    // optional int64 serverId = 4;
+    boolean hasServerId();
+    long getServerId();
+    
+    // optional string serverenCode = 5;
+    boolean hasServerenCode();
+    String getServerenCode();
+    
+    // optional int64 executeTime = 6;
+    boolean hasExecuteTime();
+    long getExecuteTime();
+    
+    // optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+    boolean hasSourceType();
+    com.alibaba.otter.canal.protocol.CanalEntry.Type getSourceType();
+    
+    // optional string schemaName = 8;
+    boolean hasSchemaName();
+    String getSchemaName();
+    
+    // optional string tableName = 9;
+    boolean hasTableName();
+    String getTableName();
+    
+    // optional int64 eventLength = 10;
+    boolean hasEventLength();
+    long getEventLength();
+    
+    // optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+    boolean hasEventType();
+    com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType();
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+    java.util.List 
+        getPropsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    int getPropsCount();
+    java.util.List 
+        getPropsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index);
+  }
+  public static final class Header extends
+      com.google.protobuf.GeneratedMessage
+      implements HeaderOrBuilder {
+    // Use Header.newBuilder() to construct.
+    private Header(Builder builder) {
+      super(builder);
+    }
+    private Header(boolean noInit) {}
+    
+    private static final Header defaultInstance;
+    public static Header getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Header getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int32 version = 1 [default = 1];
+    public static final int VERSION_FIELD_NUMBER = 1;
+    private int version_;
+    public boolean hasVersion() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public int getVersion() {
+      return version_;
+    }
+    
+    // optional string logfileName = 2;
+    public static final int LOGFILENAME_FIELD_NUMBER = 2;
+    private java.lang.Object logfileName_;
+    public boolean hasLogfileName() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getLogfileName() {
+      java.lang.Object ref = logfileName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          logfileName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getLogfileNameBytes() {
+      java.lang.Object ref = logfileName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        logfileName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 logfileOffset = 3;
+    public static final int LOGFILEOFFSET_FIELD_NUMBER = 3;
+    private long logfileOffset_;
+    public boolean hasLogfileOffset() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getLogfileOffset() {
+      return logfileOffset_;
+    }
+    
+    // optional int64 serverId = 4;
+    public static final int SERVERID_FIELD_NUMBER = 4;
+    private long serverId_;
+    public boolean hasServerId() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public long getServerId() {
+      return serverId_;
+    }
+    
+    // optional string serverenCode = 5;
+    public static final int SERVERENCODE_FIELD_NUMBER = 5;
+    private java.lang.Object serverenCode_;
+    public boolean hasServerenCode() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public String getServerenCode() {
+      java.lang.Object ref = serverenCode_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          serverenCode_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getServerenCodeBytes() {
+      java.lang.Object ref = serverenCode_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        serverenCode_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 executeTime = 6;
+    public static final int EXECUTETIME_FIELD_NUMBER = 6;
+    private long executeTime_;
+    public boolean hasExecuteTime() {
+      return ((bitField0_ & 0x00000020) == 0x00000020);
+    }
+    public long getExecuteTime() {
+      return executeTime_;
+    }
+    
+    // optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+    public static final int SOURCETYPE_FIELD_NUMBER = 7;
+    private com.alibaba.otter.canal.protocol.CanalEntry.Type sourceType_;
+    public boolean hasSourceType() {
+      return ((bitField0_ & 0x00000040) == 0x00000040);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Type getSourceType() {
+      return sourceType_;
+    }
+    
+    // optional string schemaName = 8;
+    public static final int SCHEMANAME_FIELD_NUMBER = 8;
+    private java.lang.Object schemaName_;
+    public boolean hasSchemaName() {
+      return ((bitField0_ & 0x00000080) == 0x00000080);
+    }
+    public String getSchemaName() {
+      java.lang.Object ref = schemaName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          schemaName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getSchemaNameBytes() {
+      java.lang.Object ref = schemaName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        schemaName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string tableName = 9;
+    public static final int TABLENAME_FIELD_NUMBER = 9;
+    private java.lang.Object tableName_;
+    public boolean hasTableName() {
+      return ((bitField0_ & 0x00000100) == 0x00000100);
+    }
+    public String getTableName() {
+      java.lang.Object ref = tableName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          tableName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getTableNameBytes() {
+      java.lang.Object ref = tableName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        tableName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 eventLength = 10;
+    public static final int EVENTLENGTH_FIELD_NUMBER = 10;
+    private long eventLength_;
+    public boolean hasEventLength() {
+      return ((bitField0_ & 0x00000200) == 0x00000200);
+    }
+    public long getEventLength() {
+      return eventLength_;
+    }
+    
+    // optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+    public static final int EVENTTYPE_FIELD_NUMBER = 11;
+    private com.alibaba.otter.canal.protocol.CanalEntry.EventType eventType_;
+    public boolean hasEventType() {
+      return ((bitField0_ & 0x00000400) == 0x00000400);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
+      return eventType_;
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+    public static final int PROPS_FIELD_NUMBER = 12;
+    private java.util.List props_;
+    public java.util.List getPropsList() {
+      return props_;
+    }
+    public java.util.List 
+        getPropsOrBuilderList() {
+      return props_;
+    }
+    public int getPropsCount() {
+      return props_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      return props_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index) {
+      return props_.get(index);
+    }
+    
+    private void initFields() {
+      version_ = 1;
+      logfileName_ = "";
+      logfileOffset_ = 0L;
+      serverId_ = 0L;
+      serverenCode_ = "";
+      executeTime_ = 0L;
+      sourceType_ = com.alibaba.otter.canal.protocol.CanalEntry.Type.MYSQL;
+      schemaName_ = "";
+      tableName_ = "";
+      eventLength_ = 0L;
+      eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+      props_ = java.util.Collections.emptyList();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(1, version_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getLogfileNameBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(3, logfileOffset_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeInt64(4, serverId_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBytes(5, getServerenCodeBytes());
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        output.writeInt64(6, executeTime_);
+      }
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        output.writeEnum(7, sourceType_.getNumber());
+      }
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        output.writeBytes(8, getSchemaNameBytes());
+      }
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        output.writeBytes(9, getTableNameBytes());
+      }
+      if (((bitField0_ & 0x00000200) == 0x00000200)) {
+        output.writeInt64(10, eventLength_);
+      }
+      if (((bitField0_ & 0x00000400) == 0x00000400)) {
+        output.writeEnum(11, eventType_.getNumber());
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        output.writeMessage(12, props_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(1, version_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getLogfileNameBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(3, logfileOffset_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(4, serverId_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(5, getServerenCodeBytes());
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(6, executeTime_);
+      }
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(7, sourceType_.getNumber());
+      }
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(8, getSchemaNameBytes());
+      }
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(9, getTableNameBytes());
+      }
+      if (((bitField0_ & 0x00000200) == 0x00000200)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(10, eventLength_);
+      }
+      if (((bitField0_ & 0x00000400) == 0x00000400)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(11, eventType_.getNumber());
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(12, props_.get(i));
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Header parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Header prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.HeaderOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Header.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getPropsFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        version_ = 1;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        logfileName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        logfileOffset_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        serverId_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        serverenCode_ = "";
+        bitField0_ = (bitField0_ & ~0x00000010);
+        executeTime_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000020);
+        sourceType_ = com.alibaba.otter.canal.protocol.CanalEntry.Type.MYSQL;
+        bitField0_ = (bitField0_ & ~0x00000040);
+        schemaName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000080);
+        tableName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000100);
+        eventLength_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000200);
+        eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+        bitField0_ = (bitField0_ & ~0x00000400);
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000800);
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Header.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Header getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Header build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Header result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.Header buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.Header result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Header buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Header result = new com.alibaba.otter.canal.protocol.CanalEntry.Header(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.version_ = version_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.logfileName_ = logfileName_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.logfileOffset_ = logfileOffset_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.serverId_ = serverId_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.serverenCode_ = serverenCode_;
+        if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+          to_bitField0_ |= 0x00000020;
+        }
+        result.executeTime_ = executeTime_;
+        if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+          to_bitField0_ |= 0x00000040;
+        }
+        result.sourceType_ = sourceType_;
+        if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+          to_bitField0_ |= 0x00000080;
+        }
+        result.schemaName_ = schemaName_;
+        if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+          to_bitField0_ |= 0x00000100;
+        }
+        result.tableName_ = tableName_;
+        if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+          to_bitField0_ |= 0x00000200;
+        }
+        result.eventLength_ = eventLength_;
+        if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+          to_bitField0_ |= 0x00000400;
+        }
+        result.eventType_ = eventType_;
+        if (propsBuilder_ == null) {
+          if (((bitField0_ & 0x00000800) == 0x00000800)) {
+            props_ = java.util.Collections.unmodifiableList(props_);
+            bitField0_ = (bitField0_ & ~0x00000800);
+          }
+          result.props_ = props_;
+        } else {
+          result.props_ = propsBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Header) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Header)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Header other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Header.getDefaultInstance()) return this;
+        if (other.hasVersion()) {
+          setVersion(other.getVersion());
+        }
+        if (other.hasLogfileName()) {
+          setLogfileName(other.getLogfileName());
+        }
+        if (other.hasLogfileOffset()) {
+          setLogfileOffset(other.getLogfileOffset());
+        }
+        if (other.hasServerId()) {
+          setServerId(other.getServerId());
+        }
+        if (other.hasServerenCode()) {
+          setServerenCode(other.getServerenCode());
+        }
+        if (other.hasExecuteTime()) {
+          setExecuteTime(other.getExecuteTime());
+        }
+        if (other.hasSourceType()) {
+          setSourceType(other.getSourceType());
+        }
+        if (other.hasSchemaName()) {
+          setSchemaName(other.getSchemaName());
+        }
+        if (other.hasTableName()) {
+          setTableName(other.getTableName());
+        }
+        if (other.hasEventLength()) {
+          setEventLength(other.getEventLength());
+        }
+        if (other.hasEventType()) {
+          setEventType(other.getEventType());
+        }
+        if (propsBuilder_ == null) {
+          if (!other.props_.isEmpty()) {
+            if (props_.isEmpty()) {
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000800);
+            } else {
+              ensurePropsIsMutable();
+              props_.addAll(other.props_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.props_.isEmpty()) {
+            if (propsBuilder_.isEmpty()) {
+              propsBuilder_.dispose();
+              propsBuilder_ = null;
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000800);
+              propsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getPropsFieldBuilder() : null;
+            } else {
+              propsBuilder_.addAllMessages(other.props_);
+            }
+          }
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              version_ = input.readInt32();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              logfileName_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              logfileOffset_ = input.readInt64();
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              serverId_ = input.readInt64();
+              break;
+            }
+            case 42: {
+              bitField0_ |= 0x00000010;
+              serverenCode_ = input.readBytes();
+              break;
+            }
+            case 48: {
+              bitField0_ |= 0x00000020;
+              executeTime_ = input.readInt64();
+              break;
+            }
+            case 56: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalEntry.Type value = com.alibaba.otter.canal.protocol.CanalEntry.Type.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(7, rawValue);
+              } else {
+                bitField0_ |= 0x00000040;
+                sourceType_ = value;
+              }
+              break;
+            }
+            case 66: {
+              bitField0_ |= 0x00000080;
+              schemaName_ = input.readBytes();
+              break;
+            }
+            case 74: {
+              bitField0_ |= 0x00000100;
+              tableName_ = input.readBytes();
+              break;
+            }
+            case 80: {
+              bitField0_ |= 0x00000200;
+              eventLength_ = input.readInt64();
+              break;
+            }
+            case 88: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalEntry.EventType value = com.alibaba.otter.canal.protocol.CanalEntry.EventType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(11, rawValue);
+              } else {
+                bitField0_ |= 0x00000400;
+                eventType_ = value;
+              }
+              break;
+            }
+            case 98: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addProps(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int32 version = 1 [default = 1];
+      private int version_ = 1;
+      public boolean hasVersion() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public int getVersion() {
+        return version_;
+      }
+      public Builder setVersion(int value) {
+        bitField0_ |= 0x00000001;
+        version_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearVersion() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        version_ = 1;
+        onChanged();
+        return this;
+      }
+      
+      // optional string logfileName = 2;
+      private java.lang.Object logfileName_ = "";
+      public boolean hasLogfileName() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getLogfileName() {
+        java.lang.Object ref = logfileName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          logfileName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setLogfileName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        logfileName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearLogfileName() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        logfileName_ = getDefaultInstance().getLogfileName();
+        onChanged();
+        return this;
+      }
+      void setLogfileName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        logfileName_ = value;
+        onChanged();
+      }
+      
+      // optional int64 logfileOffset = 3;
+      private long logfileOffset_ ;
+      public boolean hasLogfileOffset() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public long getLogfileOffset() {
+        return logfileOffset_;
+      }
+      public Builder setLogfileOffset(long value) {
+        bitField0_ |= 0x00000004;
+        logfileOffset_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearLogfileOffset() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        logfileOffset_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional int64 serverId = 4;
+      private long serverId_ ;
+      public boolean hasServerId() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public long getServerId() {
+        return serverId_;
+      }
+      public Builder setServerId(long value) {
+        bitField0_ |= 0x00000008;
+        serverId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearServerId() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        serverId_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional string serverenCode = 5;
+      private java.lang.Object serverenCode_ = "";
+      public boolean hasServerenCode() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public String getServerenCode() {
+        java.lang.Object ref = serverenCode_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          serverenCode_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setServerenCode(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000010;
+        serverenCode_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearServerenCode() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        serverenCode_ = getDefaultInstance().getServerenCode();
+        onChanged();
+        return this;
+      }
+      void setServerenCode(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000010;
+        serverenCode_ = value;
+        onChanged();
+      }
+      
+      // optional int64 executeTime = 6;
+      private long executeTime_ ;
+      public boolean hasExecuteTime() {
+        return ((bitField0_ & 0x00000020) == 0x00000020);
+      }
+      public long getExecuteTime() {
+        return executeTime_;
+      }
+      public Builder setExecuteTime(long value) {
+        bitField0_ |= 0x00000020;
+        executeTime_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearExecuteTime() {
+        bitField0_ = (bitField0_ & ~0x00000020);
+        executeTime_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional .com.alibaba.otter.canal.protocol.Type sourceType = 7 [default = MYSQL];
+      private com.alibaba.otter.canal.protocol.CanalEntry.Type sourceType_ = com.alibaba.otter.canal.protocol.CanalEntry.Type.MYSQL;
+      public boolean hasSourceType() {
+        return ((bitField0_ & 0x00000040) == 0x00000040);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Type getSourceType() {
+        return sourceType_;
+      }
+      public Builder setSourceType(com.alibaba.otter.canal.protocol.CanalEntry.Type value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000040;
+        sourceType_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSourceType() {
+        bitField0_ = (bitField0_ & ~0x00000040);
+        sourceType_ = com.alibaba.otter.canal.protocol.CanalEntry.Type.MYSQL;
+        onChanged();
+        return this;
+      }
+      
+      // optional string schemaName = 8;
+      private java.lang.Object schemaName_ = "";
+      public boolean hasSchemaName() {
+        return ((bitField0_ & 0x00000080) == 0x00000080);
+      }
+      public String getSchemaName() {
+        java.lang.Object ref = schemaName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          schemaName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setSchemaName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000080;
+        schemaName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSchemaName() {
+        bitField0_ = (bitField0_ & ~0x00000080);
+        schemaName_ = getDefaultInstance().getSchemaName();
+        onChanged();
+        return this;
+      }
+      void setSchemaName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000080;
+        schemaName_ = value;
+        onChanged();
+      }
+      
+      // optional string tableName = 9;
+      private java.lang.Object tableName_ = "";
+      public boolean hasTableName() {
+        return ((bitField0_ & 0x00000100) == 0x00000100);
+      }
+      public String getTableName() {
+        java.lang.Object ref = tableName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          tableName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setTableName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000100;
+        tableName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTableName() {
+        bitField0_ = (bitField0_ & ~0x00000100);
+        tableName_ = getDefaultInstance().getTableName();
+        onChanged();
+        return this;
+      }
+      void setTableName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000100;
+        tableName_ = value;
+        onChanged();
+      }
+      
+      // optional int64 eventLength = 10;
+      private long eventLength_ ;
+      public boolean hasEventLength() {
+        return ((bitField0_ & 0x00000200) == 0x00000200);
+      }
+      public long getEventLength() {
+        return eventLength_;
+      }
+      public Builder setEventLength(long value) {
+        bitField0_ |= 0x00000200;
+        eventLength_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearEventLength() {
+        bitField0_ = (bitField0_ & ~0x00000200);
+        eventLength_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional .com.alibaba.otter.canal.protocol.EventType eventType = 11 [default = UPDATE];
+      private com.alibaba.otter.canal.protocol.CanalEntry.EventType eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+      public boolean hasEventType() {
+        return ((bitField0_ & 0x00000400) == 0x00000400);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
+        return eventType_;
+      }
+      public Builder setEventType(com.alibaba.otter.canal.protocol.CanalEntry.EventType value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000400;
+        eventType_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearEventType() {
+        bitField0_ = (bitField0_ & ~0x00000400);
+        eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+        onChanged();
+        return this;
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Pair props = 12;
+      private java.util.List props_ =
+        java.util.Collections.emptyList();
+      private void ensurePropsIsMutable() {
+        if (!((bitField0_ & 0x00000800) == 0x00000800)) {
+          props_ = new java.util.ArrayList(props_);
+          bitField0_ |= 0x00000800;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      
+      public java.util.List getPropsList() {
+        if (propsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(props_);
+        } else {
+          return propsBuilder_.getMessageList();
+        }
+      }
+      public int getPropsCount() {
+        if (propsBuilder_ == null) {
+          return props_.size();
+        } else {
+          return propsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);
+        } else {
+          return propsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.set(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllProps(
+          java.lang.Iterable values) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          super.addAll(values, props_);
+          onChanged();
+        } else {
+          propsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearProps() {
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000800);
+          onChanged();
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeProps(int index) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.remove(index);
+          onChanged();
+        } else {
+          propsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+          int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);  } else {
+          return propsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getPropsOrBuilderList() {
+        if (propsBuilder_ != null) {
+          return propsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(props_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+        return getPropsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public java.util.List 
+           getPropsBuilderList() {
+        return getPropsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+          getPropsFieldBuilder() {
+        if (propsBuilder_ == null) {
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+                  props_,
+                  ((bitField0_ & 0x00000800) == 0x00000800),
+                  getParentForChildren(),
+                  isClean());
+          props_ = null;
+        }
+        return propsBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Header)
+    }
+    
+    static {
+      defaultInstance = new Header(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Header)
+  }
+  
+  public interface ColumnOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int32 index = 1;
+    boolean hasIndex();
+    int getIndex();
+    
+    // optional int32 sqlType = 2;
+    boolean hasSqlType();
+    int getSqlType();
+    
+    // optional string name = 3;
+    boolean hasName();
+    String getName();
+    
+    // optional bool isKey = 4;
+    boolean hasIsKey();
+    boolean getIsKey();
+    
+    // optional bool updated = 5;
+    boolean hasUpdated();
+    boolean getUpdated();
+    
+    // optional bool isNull = 6 [default = false];
+    boolean hasIsNull();
+    boolean getIsNull();
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+    java.util.List 
+        getPropsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    int getPropsCount();
+    java.util.List 
+        getPropsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index);
+    
+    // optional string value = 8;
+    boolean hasValue();
+    String getValue();
+    
+    // optional int32 length = 9;
+    boolean hasLength();
+    int getLength();
+    
+    // optional string mysqlType = 10;
+    boolean hasMysqlType();
+    String getMysqlType();
+  }
+  public static final class Column extends
+      com.google.protobuf.GeneratedMessage
+      implements ColumnOrBuilder {
+    // Use Column.newBuilder() to construct.
+    private Column(Builder builder) {
+      super(builder);
+    }
+    private Column(boolean noInit) {}
+    
+    private static final Column defaultInstance;
+    public static Column getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Column getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int32 index = 1;
+    public static final int INDEX_FIELD_NUMBER = 1;
+    private int index_;
+    public boolean hasIndex() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public int getIndex() {
+      return index_;
+    }
+    
+    // optional int32 sqlType = 2;
+    public static final int SQLTYPE_FIELD_NUMBER = 2;
+    private int sqlType_;
+    public boolean hasSqlType() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public int getSqlType() {
+      return sqlType_;
+    }
+    
+    // optional string name = 3;
+    public static final int NAME_FIELD_NUMBER = 3;
+    private java.lang.Object name_;
+    public boolean hasName() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public String getName() {
+      java.lang.Object ref = name_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          name_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getNameBytes() {
+      java.lang.Object ref = name_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        name_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional bool isKey = 4;
+    public static final int ISKEY_FIELD_NUMBER = 4;
+    private boolean isKey_;
+    public boolean hasIsKey() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public boolean getIsKey() {
+      return isKey_;
+    }
+    
+    // optional bool updated = 5;
+    public static final int UPDATED_FIELD_NUMBER = 5;
+    private boolean updated_;
+    public boolean hasUpdated() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public boolean getUpdated() {
+      return updated_;
+    }
+    
+    // optional bool isNull = 6 [default = false];
+    public static final int ISNULL_FIELD_NUMBER = 6;
+    private boolean isNull_;
+    public boolean hasIsNull() {
+      return ((bitField0_ & 0x00000020) == 0x00000020);
+    }
+    public boolean getIsNull() {
+      return isNull_;
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+    public static final int PROPS_FIELD_NUMBER = 7;
+    private java.util.List props_;
+    public java.util.List getPropsList() {
+      return props_;
+    }
+    public java.util.List 
+        getPropsOrBuilderList() {
+      return props_;
+    }
+    public int getPropsCount() {
+      return props_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      return props_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index) {
+      return props_.get(index);
+    }
+    
+    // optional string value = 8;
+    public static final int VALUE_FIELD_NUMBER = 8;
+    private java.lang.Object value_;
+    public boolean hasValue() {
+      return ((bitField0_ & 0x00000040) == 0x00000040);
+    }
+    public String getValue() {
+      java.lang.Object ref = value_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          value_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getValueBytes() {
+      java.lang.Object ref = value_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        value_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int32 length = 9;
+    public static final int LENGTH_FIELD_NUMBER = 9;
+    private int length_;
+    public boolean hasLength() {
+      return ((bitField0_ & 0x00000080) == 0x00000080);
+    }
+    public int getLength() {
+      return length_;
+    }
+    
+    // optional string mysqlType = 10;
+    public static final int MYSQLTYPE_FIELD_NUMBER = 10;
+    private java.lang.Object mysqlType_;
+    public boolean hasMysqlType() {
+      return ((bitField0_ & 0x00000100) == 0x00000100);
+    }
+    public String getMysqlType() {
+      java.lang.Object ref = mysqlType_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          mysqlType_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getMysqlTypeBytes() {
+      java.lang.Object ref = mysqlType_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        mysqlType_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    private void initFields() {
+      index_ = 0;
+      sqlType_ = 0;
+      name_ = "";
+      isKey_ = false;
+      updated_ = false;
+      isNull_ = false;
+      props_ = java.util.Collections.emptyList();
+      value_ = "";
+      length_ = 0;
+      mysqlType_ = "";
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(1, index_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeInt32(2, sqlType_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(3, getNameBytes());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeBool(4, isKey_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBool(5, updated_);
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        output.writeBool(6, isNull_);
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        output.writeMessage(7, props_.get(i));
+      }
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        output.writeBytes(8, getValueBytes());
+      }
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        output.writeInt32(9, length_);
+      }
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        output.writeBytes(10, getMysqlTypeBytes());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(1, index_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(2, sqlType_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(3, getNameBytes());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBoolSize(4, isKey_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBoolSize(5, updated_);
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBoolSize(6, isNull_);
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(7, props_.get(i));
+      }
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(8, getValueBytes());
+      }
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(9, length_);
+      }
+      if (((bitField0_ & 0x00000100) == 0x00000100)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(10, getMysqlTypeBytes());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Column parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Column prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Column.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getPropsFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        index_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        sqlType_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        name_ = "";
+        bitField0_ = (bitField0_ & ~0x00000004);
+        isKey_ = false;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        updated_ = false;
+        bitField0_ = (bitField0_ & ~0x00000010);
+        isNull_ = false;
+        bitField0_ = (bitField0_ & ~0x00000020);
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000040);
+        } else {
+          propsBuilder_.clear();
+        }
+        value_ = "";
+        bitField0_ = (bitField0_ & ~0x00000080);
+        length_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000100);
+        mysqlType_ = "";
+        bitField0_ = (bitField0_ & ~0x00000200);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Column.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Column result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.Column buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.Column result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Column result = new com.alibaba.otter.canal.protocol.CanalEntry.Column(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.index_ = index_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.sqlType_ = sqlType_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.name_ = name_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.isKey_ = isKey_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.updated_ = updated_;
+        if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+          to_bitField0_ |= 0x00000020;
+        }
+        result.isNull_ = isNull_;
+        if (propsBuilder_ == null) {
+          if (((bitField0_ & 0x00000040) == 0x00000040)) {
+            props_ = java.util.Collections.unmodifiableList(props_);
+            bitField0_ = (bitField0_ & ~0x00000040);
+          }
+          result.props_ = props_;
+        } else {
+          result.props_ = propsBuilder_.build();
+        }
+        if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+          to_bitField0_ |= 0x00000040;
+        }
+        result.value_ = value_;
+        if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+          to_bitField0_ |= 0x00000080;
+        }
+        result.length_ = length_;
+        if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+          to_bitField0_ |= 0x00000100;
+        }
+        result.mysqlType_ = mysqlType_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Column) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Column)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Column other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance()) return this;
+        if (other.hasIndex()) {
+          setIndex(other.getIndex());
+        }
+        if (other.hasSqlType()) {
+          setSqlType(other.getSqlType());
+        }
+        if (other.hasName()) {
+          setName(other.getName());
+        }
+        if (other.hasIsKey()) {
+          setIsKey(other.getIsKey());
+        }
+        if (other.hasUpdated()) {
+          setUpdated(other.getUpdated());
+        }
+        if (other.hasIsNull()) {
+          setIsNull(other.getIsNull());
+        }
+        if (propsBuilder_ == null) {
+          if (!other.props_.isEmpty()) {
+            if (props_.isEmpty()) {
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000040);
+            } else {
+              ensurePropsIsMutable();
+              props_.addAll(other.props_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.props_.isEmpty()) {
+            if (propsBuilder_.isEmpty()) {
+              propsBuilder_.dispose();
+              propsBuilder_ = null;
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000040);
+              propsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getPropsFieldBuilder() : null;
+            } else {
+              propsBuilder_.addAllMessages(other.props_);
+            }
+          }
+        }
+        if (other.hasValue()) {
+          setValue(other.getValue());
+        }
+        if (other.hasLength()) {
+          setLength(other.getLength());
+        }
+        if (other.hasMysqlType()) {
+          setMysqlType(other.getMysqlType());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              index_ = input.readInt32();
+              break;
+            }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              sqlType_ = input.readInt32();
+              break;
+            }
+            case 26: {
+              bitField0_ |= 0x00000004;
+              name_ = input.readBytes();
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              isKey_ = input.readBool();
+              break;
+            }
+            case 40: {
+              bitField0_ |= 0x00000010;
+              updated_ = input.readBool();
+              break;
+            }
+            case 48: {
+              bitField0_ |= 0x00000020;
+              isNull_ = input.readBool();
+              break;
+            }
+            case 58: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addProps(subBuilder.buildPartial());
+              break;
+            }
+            case 66: {
+              bitField0_ |= 0x00000080;
+              value_ = input.readBytes();
+              break;
+            }
+            case 72: {
+              bitField0_ |= 0x00000100;
+              length_ = input.readInt32();
+              break;
+            }
+            case 82: {
+              bitField0_ |= 0x00000200;
+              mysqlType_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int32 index = 1;
+      private int index_ ;
+      public boolean hasIndex() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public int getIndex() {
+        return index_;
+      }
+      public Builder setIndex(int value) {
+        bitField0_ |= 0x00000001;
+        index_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearIndex() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        index_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 sqlType = 2;
+      private int sqlType_ ;
+      public boolean hasSqlType() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public int getSqlType() {
+        return sqlType_;
+      }
+      public Builder setSqlType(int value) {
+        bitField0_ |= 0x00000002;
+        sqlType_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSqlType() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        sqlType_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional string name = 3;
+      private java.lang.Object name_ = "";
+      public boolean hasName() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public String getName() {
+        java.lang.Object ref = name_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          name_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        name_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearName() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        name_ = getDefaultInstance().getName();
+        onChanged();
+        return this;
+      }
+      void setName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000004;
+        name_ = value;
+        onChanged();
+      }
+      
+      // optional bool isKey = 4;
+      private boolean isKey_ ;
+      public boolean hasIsKey() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public boolean getIsKey() {
+        return isKey_;
+      }
+      public Builder setIsKey(boolean value) {
+        bitField0_ |= 0x00000008;
+        isKey_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearIsKey() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        isKey_ = false;
+        onChanged();
+        return this;
+      }
+      
+      // optional bool updated = 5;
+      private boolean updated_ ;
+      public boolean hasUpdated() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public boolean getUpdated() {
+        return updated_;
+      }
+      public Builder setUpdated(boolean value) {
+        bitField0_ |= 0x00000010;
+        updated_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearUpdated() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        updated_ = false;
+        onChanged();
+        return this;
+      }
+      
+      // optional bool isNull = 6 [default = false];
+      private boolean isNull_ ;
+      public boolean hasIsNull() {
+        return ((bitField0_ & 0x00000020) == 0x00000020);
+      }
+      public boolean getIsNull() {
+        return isNull_;
+      }
+      public Builder setIsNull(boolean value) {
+        bitField0_ |= 0x00000020;
+        isNull_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearIsNull() {
+        bitField0_ = (bitField0_ & ~0x00000020);
+        isNull_ = false;
+        onChanged();
+        return this;
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Pair props = 7;
+      private java.util.List props_ =
+        java.util.Collections.emptyList();
+      private void ensurePropsIsMutable() {
+        if (!((bitField0_ & 0x00000040) == 0x00000040)) {
+          props_ = new java.util.ArrayList(props_);
+          bitField0_ |= 0x00000040;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      
+      public java.util.List getPropsList() {
+        if (propsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(props_);
+        } else {
+          return propsBuilder_.getMessageList();
+        }
+      }
+      public int getPropsCount() {
+        if (propsBuilder_ == null) {
+          return props_.size();
+        } else {
+          return propsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);
+        } else {
+          return propsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.set(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllProps(
+          java.lang.Iterable values) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          super.addAll(values, props_);
+          onChanged();
+        } else {
+          propsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearProps() {
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000040);
+          onChanged();
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeProps(int index) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.remove(index);
+          onChanged();
+        } else {
+          propsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+          int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);  } else {
+          return propsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getPropsOrBuilderList() {
+        if (propsBuilder_ != null) {
+          return propsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(props_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+        return getPropsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public java.util.List 
+           getPropsBuilderList() {
+        return getPropsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+          getPropsFieldBuilder() {
+        if (propsBuilder_ == null) {
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+                  props_,
+                  ((bitField0_ & 0x00000040) == 0x00000040),
+                  getParentForChildren(),
+                  isClean());
+          props_ = null;
+        }
+        return propsBuilder_;
+      }
+      
+      // optional string value = 8;
+      private java.lang.Object value_ = "";
+      public boolean hasValue() {
+        return ((bitField0_ & 0x00000080) == 0x00000080);
+      }
+      public String getValue() {
+        java.lang.Object ref = value_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          value_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setValue(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000080;
+        value_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearValue() {
+        bitField0_ = (bitField0_ & ~0x00000080);
+        value_ = getDefaultInstance().getValue();
+        onChanged();
+        return this;
+      }
+      void setValue(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000080;
+        value_ = value;
+        onChanged();
+      }
+      
+      // optional int32 length = 9;
+      private int length_ ;
+      public boolean hasLength() {
+        return ((bitField0_ & 0x00000100) == 0x00000100);
+      }
+      public int getLength() {
+        return length_;
+      }
+      public Builder setLength(int value) {
+        bitField0_ |= 0x00000100;
+        length_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearLength() {
+        bitField0_ = (bitField0_ & ~0x00000100);
+        length_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional string mysqlType = 10;
+      private java.lang.Object mysqlType_ = "";
+      public boolean hasMysqlType() {
+        return ((bitField0_ & 0x00000200) == 0x00000200);
+      }
+      public String getMysqlType() {
+        java.lang.Object ref = mysqlType_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          mysqlType_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setMysqlType(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000200;
+        mysqlType_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearMysqlType() {
+        bitField0_ = (bitField0_ & ~0x00000200);
+        mysqlType_ = getDefaultInstance().getMysqlType();
+        onChanged();
+        return this;
+      }
+      void setMysqlType(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000200;
+        mysqlType_ = value;
+        onChanged();
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Column)
+    }
+    
+    static {
+      defaultInstance = new Column(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Column)
+  }
+  
+  public interface RowDataOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+    java.util.List 
+        getBeforeColumnsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Column getBeforeColumns(int index);
+    int getBeforeColumnsCount();
+    java.util.List 
+        getBeforeColumnsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getBeforeColumnsOrBuilder(
+        int index);
+    
+    // repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+    java.util.List 
+        getAfterColumnsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Column getAfterColumns(int index);
+    int getAfterColumnsCount();
+    java.util.List 
+        getAfterColumnsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getAfterColumnsOrBuilder(
+        int index);
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+    java.util.List 
+        getPropsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    int getPropsCount();
+    java.util.List 
+        getPropsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index);
+  }
+  public static final class RowData extends
+      com.google.protobuf.GeneratedMessage
+      implements RowDataOrBuilder {
+    // Use RowData.newBuilder() to construct.
+    private RowData(Builder builder) {
+      super(builder);
+    }
+    private RowData(boolean noInit) {}
+    
+    private static final RowData defaultInstance;
+    public static RowData getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public RowData getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable;
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+    public static final int BEFORECOLUMNS_FIELD_NUMBER = 1;
+    private java.util.List beforeColumns_;
+    public java.util.List getBeforeColumnsList() {
+      return beforeColumns_;
+    }
+    public java.util.List 
+        getBeforeColumnsOrBuilderList() {
+      return beforeColumns_;
+    }
+    public int getBeforeColumnsCount() {
+      return beforeColumns_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Column getBeforeColumns(int index) {
+      return beforeColumns_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getBeforeColumnsOrBuilder(
+        int index) {
+      return beforeColumns_.get(index);
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+    public static final int AFTERCOLUMNS_FIELD_NUMBER = 2;
+    private java.util.List afterColumns_;
+    public java.util.List getAfterColumnsList() {
+      return afterColumns_;
+    }
+    public java.util.List 
+        getAfterColumnsOrBuilderList() {
+      return afterColumns_;
+    }
+    public int getAfterColumnsCount() {
+      return afterColumns_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Column getAfterColumns(int index) {
+      return afterColumns_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getAfterColumnsOrBuilder(
+        int index) {
+      return afterColumns_.get(index);
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+    public static final int PROPS_FIELD_NUMBER = 3;
+    private java.util.List props_;
+    public java.util.List getPropsList() {
+      return props_;
+    }
+    public java.util.List 
+        getPropsOrBuilderList() {
+      return props_;
+    }
+    public int getPropsCount() {
+      return props_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      return props_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index) {
+      return props_.get(index);
+    }
+    
+    private void initFields() {
+      beforeColumns_ = java.util.Collections.emptyList();
+      afterColumns_ = java.util.Collections.emptyList();
+      props_ = java.util.Collections.emptyList();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      for (int i = 0; i < beforeColumns_.size(); i++) {
+        output.writeMessage(1, beforeColumns_.get(i));
+      }
+      for (int i = 0; i < afterColumns_.size(); i++) {
+        output.writeMessage(2, afterColumns_.get(i));
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        output.writeMessage(3, props_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      for (int i = 0; i < beforeColumns_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(1, beforeColumns_.get(i));
+      }
+      for (int i = 0; i < afterColumns_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(2, afterColumns_.get(i));
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(3, props_.get(i));
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowData parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.RowData prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.RowData.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getBeforeColumnsFieldBuilder();
+          getAfterColumnsFieldBuilder();
+          getPropsFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        if (beforeColumnsBuilder_ == null) {
+          beforeColumns_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000001);
+        } else {
+          beforeColumnsBuilder_.clear();
+        }
+        if (afterColumnsBuilder_ == null) {
+          afterColumns_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000002);
+        } else {
+          afterColumnsBuilder_.clear();
+        }
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.RowData result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.RowData buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.RowData result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.RowData result = new com.alibaba.otter.canal.protocol.CanalEntry.RowData(this);
+        int from_bitField0_ = bitField0_;
+        if (beforeColumnsBuilder_ == null) {
+          if (((bitField0_ & 0x00000001) == 0x00000001)) {
+            beforeColumns_ = java.util.Collections.unmodifiableList(beforeColumns_);
+            bitField0_ = (bitField0_ & ~0x00000001);
+          }
+          result.beforeColumns_ = beforeColumns_;
+        } else {
+          result.beforeColumns_ = beforeColumnsBuilder_.build();
+        }
+        if (afterColumnsBuilder_ == null) {
+          if (((bitField0_ & 0x00000002) == 0x00000002)) {
+            afterColumns_ = java.util.Collections.unmodifiableList(afterColumns_);
+            bitField0_ = (bitField0_ & ~0x00000002);
+          }
+          result.afterColumns_ = afterColumns_;
+        } else {
+          result.afterColumns_ = afterColumnsBuilder_.build();
+        }
+        if (propsBuilder_ == null) {
+          if (((bitField0_ & 0x00000004) == 0x00000004)) {
+            props_ = java.util.Collections.unmodifiableList(props_);
+            bitField0_ = (bitField0_ & ~0x00000004);
+          }
+          result.props_ = props_;
+        } else {
+          result.props_ = propsBuilder_.build();
+        }
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.RowData) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.RowData)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.RowData other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance()) return this;
+        if (beforeColumnsBuilder_ == null) {
+          if (!other.beforeColumns_.isEmpty()) {
+            if (beforeColumns_.isEmpty()) {
+              beforeColumns_ = other.beforeColumns_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+            } else {
+              ensureBeforeColumnsIsMutable();
+              beforeColumns_.addAll(other.beforeColumns_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.beforeColumns_.isEmpty()) {
+            if (beforeColumnsBuilder_.isEmpty()) {
+              beforeColumnsBuilder_.dispose();
+              beforeColumnsBuilder_ = null;
+              beforeColumns_ = other.beforeColumns_;
+              bitField0_ = (bitField0_ & ~0x00000001);
+              beforeColumnsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getBeforeColumnsFieldBuilder() : null;
+            } else {
+              beforeColumnsBuilder_.addAllMessages(other.beforeColumns_);
+            }
+          }
+        }
+        if (afterColumnsBuilder_ == null) {
+          if (!other.afterColumns_.isEmpty()) {
+            if (afterColumns_.isEmpty()) {
+              afterColumns_ = other.afterColumns_;
+              bitField0_ = (bitField0_ & ~0x00000002);
+            } else {
+              ensureAfterColumnsIsMutable();
+              afterColumns_.addAll(other.afterColumns_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.afterColumns_.isEmpty()) {
+            if (afterColumnsBuilder_.isEmpty()) {
+              afterColumnsBuilder_.dispose();
+              afterColumnsBuilder_ = null;
+              afterColumns_ = other.afterColumns_;
+              bitField0_ = (bitField0_ & ~0x00000002);
+              afterColumnsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getAfterColumnsFieldBuilder() : null;
+            } else {
+              afterColumnsBuilder_.addAllMessages(other.afterColumns_);
+            }
+          }
+        }
+        if (propsBuilder_ == null) {
+          if (!other.props_.isEmpty()) {
+            if (props_.isEmpty()) {
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000004);
+            } else {
+              ensurePropsIsMutable();
+              props_.addAll(other.props_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.props_.isEmpty()) {
+            if (propsBuilder_.isEmpty()) {
+              propsBuilder_.dispose();
+              propsBuilder_ = null;
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000004);
+              propsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getPropsFieldBuilder() : null;
+            } else {
+              propsBuilder_.addAllMessages(other.props_);
+            }
+          }
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Column.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addBeforeColumns(subBuilder.buildPartial());
+              break;
+            }
+            case 18: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Column.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addAfterColumns(subBuilder.buildPartial());
+              break;
+            }
+            case 26: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addProps(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // repeated .com.alibaba.otter.canal.protocol.Column beforeColumns = 1;
+      private java.util.List beforeColumns_ =
+        java.util.Collections.emptyList();
+      private void ensureBeforeColumnsIsMutable() {
+        if (!((bitField0_ & 0x00000001) == 0x00000001)) {
+          beforeColumns_ = new java.util.ArrayList(beforeColumns_);
+          bitField0_ |= 0x00000001;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> beforeColumnsBuilder_;
+      
+      public java.util.List getBeforeColumnsList() {
+        if (beforeColumnsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(beforeColumns_);
+        } else {
+          return beforeColumnsBuilder_.getMessageList();
+        }
+      }
+      public int getBeforeColumnsCount() {
+        if (beforeColumnsBuilder_ == null) {
+          return beforeColumns_.size();
+        } else {
+          return beforeColumnsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column getBeforeColumns(int index) {
+        if (beforeColumnsBuilder_ == null) {
+          return beforeColumns_.get(index);
+        } else {
+          return beforeColumnsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setBeforeColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+        if (beforeColumnsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.set(index, value);
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setBeforeColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+        if (beforeColumnsBuilder_ == null) {
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addBeforeColumns(com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+        if (beforeColumnsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.add(value);
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addBeforeColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+        if (beforeColumnsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.add(index, value);
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addBeforeColumns(
+          com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+        if (beforeColumnsBuilder_ == null) {
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.add(builderForValue.build());
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addBeforeColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+        if (beforeColumnsBuilder_ == null) {
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllBeforeColumns(
+          java.lang.Iterable values) {
+        if (beforeColumnsBuilder_ == null) {
+          ensureBeforeColumnsIsMutable();
+          super.addAll(values, beforeColumns_);
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearBeforeColumns() {
+        if (beforeColumnsBuilder_ == null) {
+          beforeColumns_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000001);
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeBeforeColumns(int index) {
+        if (beforeColumnsBuilder_ == null) {
+          ensureBeforeColumnsIsMutable();
+          beforeColumns_.remove(index);
+          onChanged();
+        } else {
+          beforeColumnsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder getBeforeColumnsBuilder(
+          int index) {
+        return getBeforeColumnsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getBeforeColumnsOrBuilder(
+          int index) {
+        if (beforeColumnsBuilder_ == null) {
+          return beforeColumns_.get(index);  } else {
+          return beforeColumnsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getBeforeColumnsOrBuilderList() {
+        if (beforeColumnsBuilder_ != null) {
+          return beforeColumnsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(beforeColumns_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addBeforeColumnsBuilder() {
+        return getBeforeColumnsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addBeforeColumnsBuilder(
+          int index) {
+        return getBeforeColumnsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+      }
+      public java.util.List 
+           getBeforeColumnsBuilderList() {
+        return getBeforeColumnsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> 
+          getBeforeColumnsFieldBuilder() {
+        if (beforeColumnsBuilder_ == null) {
+          beforeColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder>(
+                  beforeColumns_,
+                  ((bitField0_ & 0x00000001) == 0x00000001),
+                  getParentForChildren(),
+                  isClean());
+          beforeColumns_ = null;
+        }
+        return beforeColumnsBuilder_;
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Column afterColumns = 2;
+      private java.util.List afterColumns_ =
+        java.util.Collections.emptyList();
+      private void ensureAfterColumnsIsMutable() {
+        if (!((bitField0_ & 0x00000002) == 0x00000002)) {
+          afterColumns_ = new java.util.ArrayList(afterColumns_);
+          bitField0_ |= 0x00000002;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> afterColumnsBuilder_;
+      
+      public java.util.List getAfterColumnsList() {
+        if (afterColumnsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(afterColumns_);
+        } else {
+          return afterColumnsBuilder_.getMessageList();
+        }
+      }
+      public int getAfterColumnsCount() {
+        if (afterColumnsBuilder_ == null) {
+          return afterColumns_.size();
+        } else {
+          return afterColumnsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column getAfterColumns(int index) {
+        if (afterColumnsBuilder_ == null) {
+          return afterColumns_.get(index);
+        } else {
+          return afterColumnsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setAfterColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+        if (afterColumnsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureAfterColumnsIsMutable();
+          afterColumns_.set(index, value);
+          onChanged();
+        } else {
+          afterColumnsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setAfterColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+        if (afterColumnsBuilder_ == null) {
+          ensureAfterColumnsIsMutable();
+          afterColumns_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          afterColumnsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAfterColumns(com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+        if (afterColumnsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureAfterColumnsIsMutable();
+          afterColumns_.add(value);
+          onChanged();
+        } else {
+          afterColumnsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addAfterColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column value) {
+        if (afterColumnsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureAfterColumnsIsMutable();
+          afterColumns_.add(index, value);
+          onChanged();
+        } else {
+          afterColumnsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addAfterColumns(
+          com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+        if (afterColumnsBuilder_ == null) {
+          ensureAfterColumnsIsMutable();
+          afterColumns_.add(builderForValue.build());
+          onChanged();
+        } else {
+          afterColumnsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAfterColumns(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder builderForValue) {
+        if (afterColumnsBuilder_ == null) {
+          ensureAfterColumnsIsMutable();
+          afterColumns_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          afterColumnsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllAfterColumns(
+          java.lang.Iterable values) {
+        if (afterColumnsBuilder_ == null) {
+          ensureAfterColumnsIsMutable();
+          super.addAll(values, afterColumns_);
+          onChanged();
+        } else {
+          afterColumnsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearAfterColumns() {
+        if (afterColumnsBuilder_ == null) {
+          afterColumns_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000002);
+          onChanged();
+        } else {
+          afterColumnsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeAfterColumns(int index) {
+        if (afterColumnsBuilder_ == null) {
+          ensureAfterColumnsIsMutable();
+          afterColumns_.remove(index);
+          onChanged();
+        } else {
+          afterColumnsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder getAfterColumnsBuilder(
+          int index) {
+        return getAfterColumnsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder getAfterColumnsOrBuilder(
+          int index) {
+        if (afterColumnsBuilder_ == null) {
+          return afterColumns_.get(index);  } else {
+          return afterColumnsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getAfterColumnsOrBuilderList() {
+        if (afterColumnsBuilder_ != null) {
+          return afterColumnsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(afterColumns_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addAfterColumnsBuilder() {
+        return getAfterColumnsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder addAfterColumnsBuilder(
+          int index) {
+        return getAfterColumnsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Column.getDefaultInstance());
+      }
+      public java.util.List 
+           getAfterColumnsBuilderList() {
+        return getAfterColumnsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder> 
+          getAfterColumnsFieldBuilder() {
+        if (afterColumnsBuilder_ == null) {
+          afterColumnsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Column, com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder, com.alibaba.otter.canal.protocol.CanalEntry.ColumnOrBuilder>(
+                  afterColumns_,
+                  ((bitField0_ & 0x00000002) == 0x00000002),
+                  getParentForChildren(),
+                  isClean());
+          afterColumns_ = null;
+        }
+        return afterColumnsBuilder_;
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+      private java.util.List props_ =
+        java.util.Collections.emptyList();
+      private void ensurePropsIsMutable() {
+        if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+          props_ = new java.util.ArrayList(props_);
+          bitField0_ |= 0x00000004;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      
+      public java.util.List getPropsList() {
+        if (propsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(props_);
+        } else {
+          return propsBuilder_.getMessageList();
+        }
+      }
+      public int getPropsCount() {
+        if (propsBuilder_ == null) {
+          return props_.size();
+        } else {
+          return propsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);
+        } else {
+          return propsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.set(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllProps(
+          java.lang.Iterable values) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          super.addAll(values, props_);
+          onChanged();
+        } else {
+          propsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearProps() {
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+          onChanged();
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeProps(int index) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.remove(index);
+          onChanged();
+        } else {
+          propsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+          int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);  } else {
+          return propsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getPropsOrBuilderList() {
+        if (propsBuilder_ != null) {
+          return propsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(props_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+        return getPropsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public java.util.List 
+           getPropsBuilderList() {
+        return getPropsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+          getPropsFieldBuilder() {
+        if (propsBuilder_ == null) {
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+                  props_,
+                  ((bitField0_ & 0x00000004) == 0x00000004),
+                  getParentForChildren(),
+                  isClean());
+          props_ = null;
+        }
+        return propsBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.RowData)
+    }
+    
+    static {
+      defaultInstance = new RowData(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.RowData)
+  }
+  
+  public interface RowChangeOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int64 tableId = 1;
+    boolean hasTableId();
+    long getTableId();
+    
+    // optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+    boolean hasEventType();
+    com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType();
+    
+    // optional bool isDdl = 10 [default = false];
+    boolean hasIsDdl();
+    boolean getIsDdl();
+    
+    // optional string sql = 11;
+    boolean hasSql();
+    String getSql();
+    
+    // repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+    java.util.List 
+        getRowDatasList();
+    com.alibaba.otter.canal.protocol.CanalEntry.RowData getRowDatas(int index);
+    int getRowDatasCount();
+    java.util.List 
+        getRowDatasOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder getRowDatasOrBuilder(
+        int index);
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+    java.util.List 
+        getPropsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    int getPropsCount();
+    java.util.List 
+        getPropsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index);
+    
+    // optional string ddlSchemaName = 14;
+    boolean hasDdlSchemaName();
+    String getDdlSchemaName();
+  }
+  public static final class RowChange extends
+      com.google.protobuf.GeneratedMessage
+      implements RowChangeOrBuilder {
+    // Use RowChange.newBuilder() to construct.
+    private RowChange(Builder builder) {
+      super(builder);
+    }
+    private RowChange(boolean noInit) {}
+    
+    private static final RowChange defaultInstance;
+    public static RowChange getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public RowChange getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int64 tableId = 1;
+    public static final int TABLEID_FIELD_NUMBER = 1;
+    private long tableId_;
+    public boolean hasTableId() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public long getTableId() {
+      return tableId_;
+    }
+    
+    // optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+    public static final int EVENTTYPE_FIELD_NUMBER = 2;
+    private com.alibaba.otter.canal.protocol.CanalEntry.EventType eventType_;
+    public boolean hasEventType() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
+      return eventType_;
+    }
+    
+    // optional bool isDdl = 10 [default = false];
+    public static final int ISDDL_FIELD_NUMBER = 10;
+    private boolean isDdl_;
+    public boolean hasIsDdl() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public boolean getIsDdl() {
+      return isDdl_;
+    }
+    
+    // optional string sql = 11;
+    public static final int SQL_FIELD_NUMBER = 11;
+    private java.lang.Object sql_;
+    public boolean hasSql() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public String getSql() {
+      java.lang.Object ref = sql_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          sql_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getSqlBytes() {
+      java.lang.Object ref = sql_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        sql_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+    public static final int ROWDATAS_FIELD_NUMBER = 12;
+    private java.util.List rowDatas_;
+    public java.util.List getRowDatasList() {
+      return rowDatas_;
+    }
+    public java.util.List 
+        getRowDatasOrBuilderList() {
+      return rowDatas_;
+    }
+    public int getRowDatasCount() {
+      return rowDatas_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.RowData getRowDatas(int index) {
+      return rowDatas_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder getRowDatasOrBuilder(
+        int index) {
+      return rowDatas_.get(index);
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+    public static final int PROPS_FIELD_NUMBER = 13;
+    private java.util.List props_;
+    public java.util.List getPropsList() {
+      return props_;
+    }
+    public java.util.List 
+        getPropsOrBuilderList() {
+      return props_;
+    }
+    public int getPropsCount() {
+      return props_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      return props_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index) {
+      return props_.get(index);
+    }
+    
+    // optional string ddlSchemaName = 14;
+    public static final int DDLSCHEMANAME_FIELD_NUMBER = 14;
+    private java.lang.Object ddlSchemaName_;
+    public boolean hasDdlSchemaName() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public String getDdlSchemaName() {
+      java.lang.Object ref = ddlSchemaName_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          ddlSchemaName_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDdlSchemaNameBytes() {
+      java.lang.Object ref = ddlSchemaName_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        ddlSchemaName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    private void initFields() {
+      tableId_ = 0L;
+      eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+      isDdl_ = false;
+      sql_ = "";
+      rowDatas_ = java.util.Collections.emptyList();
+      props_ = java.util.Collections.emptyList();
+      ddlSchemaName_ = "";
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt64(1, tableId_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeEnum(2, eventType_.getNumber());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBool(10, isDdl_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeBytes(11, getSqlBytes());
+      }
+      for (int i = 0; i < rowDatas_.size(); i++) {
+        output.writeMessage(12, rowDatas_.get(i));
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        output.writeMessage(13, props_.get(i));
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBytes(14, getDdlSchemaNameBytes());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(1, tableId_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(2, eventType_.getNumber());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBoolSize(10, isDdl_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(11, getSqlBytes());
+      }
+      for (int i = 0; i < rowDatas_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(12, rowDatas_.get(i));
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(13, props_.get(i));
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(14, getDdlSchemaNameBytes());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.RowChange parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.RowChange prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.RowChangeOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.RowChange.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getRowDatasFieldBuilder();
+          getPropsFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        tableId_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        isDdl_ = false;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        sql_ = "";
+        bitField0_ = (bitField0_ & ~0x00000008);
+        if (rowDatasBuilder_ == null) {
+          rowDatas_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000010);
+        } else {
+          rowDatasBuilder_.clear();
+        }
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000020);
+        } else {
+          propsBuilder_.clear();
+        }
+        ddlSchemaName_ = "";
+        bitField0_ = (bitField0_ & ~0x00000040);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.RowChange.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowChange getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.RowChange.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowChange build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.RowChange result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.RowChange buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.RowChange result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowChange buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.RowChange result = new com.alibaba.otter.canal.protocol.CanalEntry.RowChange(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.tableId_ = tableId_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.eventType_ = eventType_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.isDdl_ = isDdl_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.sql_ = sql_;
+        if (rowDatasBuilder_ == null) {
+          if (((bitField0_ & 0x00000010) == 0x00000010)) {
+            rowDatas_ = java.util.Collections.unmodifiableList(rowDatas_);
+            bitField0_ = (bitField0_ & ~0x00000010);
+          }
+          result.rowDatas_ = rowDatas_;
+        } else {
+          result.rowDatas_ = rowDatasBuilder_.build();
+        }
+        if (propsBuilder_ == null) {
+          if (((bitField0_ & 0x00000020) == 0x00000020)) {
+            props_ = java.util.Collections.unmodifiableList(props_);
+            bitField0_ = (bitField0_ & ~0x00000020);
+          }
+          result.props_ = props_;
+        } else {
+          result.props_ = propsBuilder_.build();
+        }
+        if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.ddlSchemaName_ = ddlSchemaName_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.RowChange) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.RowChange)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.RowChange other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.RowChange.getDefaultInstance()) return this;
+        if (other.hasTableId()) {
+          setTableId(other.getTableId());
+        }
+        if (other.hasEventType()) {
+          setEventType(other.getEventType());
+        }
+        if (other.hasIsDdl()) {
+          setIsDdl(other.getIsDdl());
+        }
+        if (other.hasSql()) {
+          setSql(other.getSql());
+        }
+        if (rowDatasBuilder_ == null) {
+          if (!other.rowDatas_.isEmpty()) {
+            if (rowDatas_.isEmpty()) {
+              rowDatas_ = other.rowDatas_;
+              bitField0_ = (bitField0_ & ~0x00000010);
+            } else {
+              ensureRowDatasIsMutable();
+              rowDatas_.addAll(other.rowDatas_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.rowDatas_.isEmpty()) {
+            if (rowDatasBuilder_.isEmpty()) {
+              rowDatasBuilder_.dispose();
+              rowDatasBuilder_ = null;
+              rowDatas_ = other.rowDatas_;
+              bitField0_ = (bitField0_ & ~0x00000010);
+              rowDatasBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getRowDatasFieldBuilder() : null;
+            } else {
+              rowDatasBuilder_.addAllMessages(other.rowDatas_);
+            }
+          }
+        }
+        if (propsBuilder_ == null) {
+          if (!other.props_.isEmpty()) {
+            if (props_.isEmpty()) {
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000020);
+            } else {
+              ensurePropsIsMutable();
+              props_.addAll(other.props_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.props_.isEmpty()) {
+            if (propsBuilder_.isEmpty()) {
+              propsBuilder_.dispose();
+              propsBuilder_ = null;
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000020);
+              propsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getPropsFieldBuilder() : null;
+            } else {
+              propsBuilder_.addAllMessages(other.props_);
+            }
+          }
+        }
+        if (other.hasDdlSchemaName()) {
+          setDdlSchemaName(other.getDdlSchemaName());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              tableId_ = input.readInt64();
+              break;
+            }
+            case 16: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalEntry.EventType value = com.alibaba.otter.canal.protocol.CanalEntry.EventType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(2, rawValue);
+              } else {
+                bitField0_ |= 0x00000002;
+                eventType_ = value;
+              }
+              break;
+            }
+            case 80: {
+              bitField0_ |= 0x00000004;
+              isDdl_ = input.readBool();
+              break;
+            }
+            case 90: {
+              bitField0_ |= 0x00000008;
+              sql_ = input.readBytes();
+              break;
+            }
+            case 98: {
+              com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.RowData.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addRowDatas(subBuilder.buildPartial());
+              break;
+            }
+            case 106: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addProps(subBuilder.buildPartial());
+              break;
+            }
+            case 114: {
+              bitField0_ |= 0x00000040;
+              ddlSchemaName_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int64 tableId = 1;
+      private long tableId_ ;
+      public boolean hasTableId() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public long getTableId() {
+        return tableId_;
+      }
+      public Builder setTableId(long value) {
+        bitField0_ |= 0x00000001;
+        tableId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTableId() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        tableId_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional .com.alibaba.otter.canal.protocol.EventType eventType = 2 [default = UPDATE];
+      private com.alibaba.otter.canal.protocol.CanalEntry.EventType eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+      public boolean hasEventType() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.EventType getEventType() {
+        return eventType_;
+      }
+      public Builder setEventType(com.alibaba.otter.canal.protocol.CanalEntry.EventType value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000002;
+        eventType_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearEventType() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        eventType_ = com.alibaba.otter.canal.protocol.CanalEntry.EventType.UPDATE;
+        onChanged();
+        return this;
+      }
+      
+      // optional bool isDdl = 10 [default = false];
+      private boolean isDdl_ ;
+      public boolean hasIsDdl() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public boolean getIsDdl() {
+        return isDdl_;
+      }
+      public Builder setIsDdl(boolean value) {
+        bitField0_ |= 0x00000004;
+        isDdl_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearIsDdl() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        isDdl_ = false;
+        onChanged();
+        return this;
+      }
+      
+      // optional string sql = 11;
+      private java.lang.Object sql_ = "";
+      public boolean hasSql() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public String getSql() {
+        java.lang.Object ref = sql_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          sql_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setSql(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000008;
+        sql_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSql() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        sql_ = getDefaultInstance().getSql();
+        onChanged();
+        return this;
+      }
+      void setSql(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000008;
+        sql_ = value;
+        onChanged();
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.RowData rowDatas = 12;
+      private java.util.List rowDatas_ =
+        java.util.Collections.emptyList();
+      private void ensureRowDatasIsMutable() {
+        if (!((bitField0_ & 0x00000010) == 0x00000010)) {
+          rowDatas_ = new java.util.ArrayList(rowDatas_);
+          bitField0_ |= 0x00000010;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.RowData, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder, com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder> rowDatasBuilder_;
+      
+      public java.util.List getRowDatasList() {
+        if (rowDatasBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(rowDatas_);
+        } else {
+          return rowDatasBuilder_.getMessageList();
+        }
+      }
+      public int getRowDatasCount() {
+        if (rowDatasBuilder_ == null) {
+          return rowDatas_.size();
+        } else {
+          return rowDatasBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData getRowDatas(int index) {
+        if (rowDatasBuilder_ == null) {
+          return rowDatas_.get(index);
+        } else {
+          return rowDatasBuilder_.getMessage(index);
+        }
+      }
+      public Builder setRowDatas(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData value) {
+        if (rowDatasBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureRowDatasIsMutable();
+          rowDatas_.set(index, value);
+          onChanged();
+        } else {
+          rowDatasBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setRowDatas(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder builderForValue) {
+        if (rowDatasBuilder_ == null) {
+          ensureRowDatasIsMutable();
+          rowDatas_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          rowDatasBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addRowDatas(com.alibaba.otter.canal.protocol.CanalEntry.RowData value) {
+        if (rowDatasBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureRowDatasIsMutable();
+          rowDatas_.add(value);
+          onChanged();
+        } else {
+          rowDatasBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addRowDatas(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData value) {
+        if (rowDatasBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensureRowDatasIsMutable();
+          rowDatas_.add(index, value);
+          onChanged();
+        } else {
+          rowDatasBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addRowDatas(
+          com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder builderForValue) {
+        if (rowDatasBuilder_ == null) {
+          ensureRowDatasIsMutable();
+          rowDatas_.add(builderForValue.build());
+          onChanged();
+        } else {
+          rowDatasBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addRowDatas(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder builderForValue) {
+        if (rowDatasBuilder_ == null) {
+          ensureRowDatasIsMutable();
+          rowDatas_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          rowDatasBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllRowDatas(
+          java.lang.Iterable values) {
+        if (rowDatasBuilder_ == null) {
+          ensureRowDatasIsMutable();
+          super.addAll(values, rowDatas_);
+          onChanged();
+        } else {
+          rowDatasBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearRowDatas() {
+        if (rowDatasBuilder_ == null) {
+          rowDatas_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000010);
+          onChanged();
+        } else {
+          rowDatasBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeRowDatas(int index) {
+        if (rowDatasBuilder_ == null) {
+          ensureRowDatasIsMutable();
+          rowDatas_.remove(index);
+          onChanged();
+        } else {
+          rowDatasBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder getRowDatasBuilder(
+          int index) {
+        return getRowDatasFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder getRowDatasOrBuilder(
+          int index) {
+        if (rowDatasBuilder_ == null) {
+          return rowDatas_.get(index);  } else {
+          return rowDatasBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getRowDatasOrBuilderList() {
+        if (rowDatasBuilder_ != null) {
+          return rowDatasBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(rowDatas_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder addRowDatasBuilder() {
+        return getRowDatasFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder addRowDatasBuilder(
+          int index) {
+        return getRowDatasFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.RowData.getDefaultInstance());
+      }
+      public java.util.List 
+           getRowDatasBuilderList() {
+        return getRowDatasFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.RowData, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder, com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder> 
+          getRowDatasFieldBuilder() {
+        if (rowDatasBuilder_ == null) {
+          rowDatasBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.RowData, com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder, com.alibaba.otter.canal.protocol.CanalEntry.RowDataOrBuilder>(
+                  rowDatas_,
+                  ((bitField0_ & 0x00000010) == 0x00000010),
+                  getParentForChildren(),
+                  isClean());
+          rowDatas_ = null;
+        }
+        return rowDatasBuilder_;
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Pair props = 13;
+      private java.util.List props_ =
+        java.util.Collections.emptyList();
+      private void ensurePropsIsMutable() {
+        if (!((bitField0_ & 0x00000020) == 0x00000020)) {
+          props_ = new java.util.ArrayList(props_);
+          bitField0_ |= 0x00000020;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      
+      public java.util.List getPropsList() {
+        if (propsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(props_);
+        } else {
+          return propsBuilder_.getMessageList();
+        }
+      }
+      public int getPropsCount() {
+        if (propsBuilder_ == null) {
+          return props_.size();
+        } else {
+          return propsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);
+        } else {
+          return propsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.set(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllProps(
+          java.lang.Iterable values) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          super.addAll(values, props_);
+          onChanged();
+        } else {
+          propsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearProps() {
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000020);
+          onChanged();
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeProps(int index) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.remove(index);
+          onChanged();
+        } else {
+          propsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+          int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);  } else {
+          return propsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getPropsOrBuilderList() {
+        if (propsBuilder_ != null) {
+          return propsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(props_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+        return getPropsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public java.util.List 
+           getPropsBuilderList() {
+        return getPropsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+          getPropsFieldBuilder() {
+        if (propsBuilder_ == null) {
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+                  props_,
+                  ((bitField0_ & 0x00000020) == 0x00000020),
+                  getParentForChildren(),
+                  isClean());
+          props_ = null;
+        }
+        return propsBuilder_;
+      }
+      
+      // optional string ddlSchemaName = 14;
+      private java.lang.Object ddlSchemaName_ = "";
+      public boolean hasDdlSchemaName() {
+        return ((bitField0_ & 0x00000040) == 0x00000040);
+      }
+      public String getDdlSchemaName() {
+        java.lang.Object ref = ddlSchemaName_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          ddlSchemaName_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDdlSchemaName(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000040;
+        ddlSchemaName_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDdlSchemaName() {
+        bitField0_ = (bitField0_ & ~0x00000040);
+        ddlSchemaName_ = getDefaultInstance().getDdlSchemaName();
+        onChanged();
+        return this;
+      }
+      void setDdlSchemaName(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000040;
+        ddlSchemaName_ = value;
+        onChanged();
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.RowChange)
+    }
+    
+    static {
+      defaultInstance = new RowChange(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.RowChange)
+  }
+  
+  public interface TransactionBeginOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int64 executeTime = 1;
+    boolean hasExecuteTime();
+    long getExecuteTime();
+    
+    // optional string transactionId = 2;
+    boolean hasTransactionId();
+    String getTransactionId();
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+    java.util.List 
+        getPropsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    int getPropsCount();
+    java.util.List 
+        getPropsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index);
+    
+    // optional int64 threadId = 4;
+    boolean hasThreadId();
+    long getThreadId();
+  }
+  public static final class TransactionBegin extends
+      com.google.protobuf.GeneratedMessage
+      implements TransactionBeginOrBuilder {
+    // Use TransactionBegin.newBuilder() to construct.
+    private TransactionBegin(Builder builder) {
+      super(builder);
+    }
+    private TransactionBegin(boolean noInit) {}
+    
+    private static final TransactionBegin defaultInstance;
+    public static TransactionBegin getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public TransactionBegin getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int64 executeTime = 1;
+    public static final int EXECUTETIME_FIELD_NUMBER = 1;
+    private long executeTime_;
+    public boolean hasExecuteTime() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public long getExecuteTime() {
+      return executeTime_;
+    }
+    
+    // optional string transactionId = 2;
+    public static final int TRANSACTIONID_FIELD_NUMBER = 2;
+    private java.lang.Object transactionId_;
+    public boolean hasTransactionId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getTransactionId() {
+      java.lang.Object ref = transactionId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          transactionId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getTransactionIdBytes() {
+      java.lang.Object ref = transactionId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        transactionId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+    public static final int PROPS_FIELD_NUMBER = 3;
+    private java.util.List props_;
+    public java.util.List getPropsList() {
+      return props_;
+    }
+    public java.util.List 
+        getPropsOrBuilderList() {
+      return props_;
+    }
+    public int getPropsCount() {
+      return props_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      return props_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index) {
+      return props_.get(index);
+    }
+    
+    // optional int64 threadId = 4;
+    public static final int THREADID_FIELD_NUMBER = 4;
+    private long threadId_;
+    public boolean hasThreadId() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getThreadId() {
+      return threadId_;
+    }
+    
+    private void initFields() {
+      executeTime_ = 0L;
+      transactionId_ = "";
+      props_ = java.util.Collections.emptyList();
+      threadId_ = 0L;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt64(1, executeTime_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getTransactionIdBytes());
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        output.writeMessage(3, props_.get(i));
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(4, threadId_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(1, executeTime_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getTransactionIdBytes());
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(3, props_.get(i));
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(4, threadId_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.TransactionBeginOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getPropsFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        executeTime_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        transactionId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+        } else {
+          propsBuilder_.clear();
+        }
+        threadId_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin result = new com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.executeTime_ = executeTime_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.transactionId_ = transactionId_;
+        if (propsBuilder_ == null) {
+          if (((bitField0_ & 0x00000004) == 0x00000004)) {
+            props_ = java.util.Collections.unmodifiableList(props_);
+            bitField0_ = (bitField0_ & ~0x00000004);
+          }
+          result.props_ = props_;
+        } else {
+          result.props_ = propsBuilder_.build();
+        }
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.threadId_ = threadId_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.getDefaultInstance()) return this;
+        if (other.hasExecuteTime()) {
+          setExecuteTime(other.getExecuteTime());
+        }
+        if (other.hasTransactionId()) {
+          setTransactionId(other.getTransactionId());
+        }
+        if (propsBuilder_ == null) {
+          if (!other.props_.isEmpty()) {
+            if (props_.isEmpty()) {
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000004);
+            } else {
+              ensurePropsIsMutable();
+              props_.addAll(other.props_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.props_.isEmpty()) {
+            if (propsBuilder_.isEmpty()) {
+              propsBuilder_.dispose();
+              propsBuilder_ = null;
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000004);
+              propsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getPropsFieldBuilder() : null;
+            } else {
+              propsBuilder_.addAllMessages(other.props_);
+            }
+          }
+        }
+        if (other.hasThreadId()) {
+          setThreadId(other.getThreadId());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              executeTime_ = input.readInt64();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              transactionId_ = input.readBytes();
+              break;
+            }
+            case 26: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addProps(subBuilder.buildPartial());
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              threadId_ = input.readInt64();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int64 executeTime = 1;
+      private long executeTime_ ;
+      public boolean hasExecuteTime() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public long getExecuteTime() {
+        return executeTime_;
+      }
+      public Builder setExecuteTime(long value) {
+        bitField0_ |= 0x00000001;
+        executeTime_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearExecuteTime() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        executeTime_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional string transactionId = 2;
+      private java.lang.Object transactionId_ = "";
+      public boolean hasTransactionId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getTransactionId() {
+        java.lang.Object ref = transactionId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          transactionId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setTransactionId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        transactionId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTransactionId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        transactionId_ = getDefaultInstance().getTransactionId();
+        onChanged();
+        return this;
+      }
+      void setTransactionId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        transactionId_ = value;
+        onChanged();
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+      private java.util.List props_ =
+        java.util.Collections.emptyList();
+      private void ensurePropsIsMutable() {
+        if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+          props_ = new java.util.ArrayList(props_);
+          bitField0_ |= 0x00000004;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      
+      public java.util.List getPropsList() {
+        if (propsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(props_);
+        } else {
+          return propsBuilder_.getMessageList();
+        }
+      }
+      public int getPropsCount() {
+        if (propsBuilder_ == null) {
+          return props_.size();
+        } else {
+          return propsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);
+        } else {
+          return propsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.set(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllProps(
+          java.lang.Iterable values) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          super.addAll(values, props_);
+          onChanged();
+        } else {
+          propsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearProps() {
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+          onChanged();
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeProps(int index) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.remove(index);
+          onChanged();
+        } else {
+          propsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+          int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);  } else {
+          return propsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getPropsOrBuilderList() {
+        if (propsBuilder_ != null) {
+          return propsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(props_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+        return getPropsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public java.util.List 
+           getPropsBuilderList() {
+        return getPropsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+          getPropsFieldBuilder() {
+        if (propsBuilder_ == null) {
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+                  props_,
+                  ((bitField0_ & 0x00000004) == 0x00000004),
+                  getParentForChildren(),
+                  isClean());
+          props_ = null;
+        }
+        return propsBuilder_;
+      }
+      
+      // optional int64 threadId = 4;
+      private long threadId_ ;
+      public boolean hasThreadId() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public long getThreadId() {
+        return threadId_;
+      }
+      public Builder setThreadId(long value) {
+        bitField0_ |= 0x00000008;
+        threadId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearThreadId() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        threadId_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.TransactionBegin)
+    }
+    
+    static {
+      defaultInstance = new TransactionBegin(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.TransactionBegin)
+  }
+  
+  public interface TransactionEndOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int64 executeTime = 1;
+    boolean hasExecuteTime();
+    long getExecuteTime();
+    
+    // optional string transactionId = 2;
+    boolean hasTransactionId();
+    String getTransactionId();
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+    java.util.List 
+        getPropsList();
+    com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index);
+    int getPropsCount();
+    java.util.List 
+        getPropsOrBuilderList();
+    com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index);
+  }
+  public static final class TransactionEnd extends
+      com.google.protobuf.GeneratedMessage
+      implements TransactionEndOrBuilder {
+    // Use TransactionEnd.newBuilder() to construct.
+    private TransactionEnd(Builder builder) {
+      super(builder);
+    }
+    private TransactionEnd(boolean noInit) {}
+    
+    private static final TransactionEnd defaultInstance;
+    public static TransactionEnd getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public TransactionEnd getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int64 executeTime = 1;
+    public static final int EXECUTETIME_FIELD_NUMBER = 1;
+    private long executeTime_;
+    public boolean hasExecuteTime() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public long getExecuteTime() {
+      return executeTime_;
+    }
+    
+    // optional string transactionId = 2;
+    public static final int TRANSACTIONID_FIELD_NUMBER = 2;
+    private java.lang.Object transactionId_;
+    public boolean hasTransactionId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getTransactionId() {
+      java.lang.Object ref = transactionId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          transactionId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getTransactionIdBytes() {
+      java.lang.Object ref = transactionId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        transactionId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+    public static final int PROPS_FIELD_NUMBER = 3;
+    private java.util.List props_;
+    public java.util.List getPropsList() {
+      return props_;
+    }
+    public java.util.List 
+        getPropsOrBuilderList() {
+      return props_;
+    }
+    public int getPropsCount() {
+      return props_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+      return props_.get(index);
+    }
+    public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+        int index) {
+      return props_.get(index);
+    }
+    
+    private void initFields() {
+      executeTime_ = 0L;
+      transactionId_ = "";
+      props_ = java.util.Collections.emptyList();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt64(1, executeTime_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getTransactionIdBytes());
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        output.writeMessage(3, props_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(1, executeTime_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getTransactionIdBytes());
+      }
+      for (int i = 0; i < props_.size(); i++) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(3, props_.get(i));
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.TransactionEndOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+          getPropsFieldBuilder();
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        executeTime_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        transactionId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd result = new com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.executeTime_ = executeTime_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.transactionId_ = transactionId_;
+        if (propsBuilder_ == null) {
+          if (((bitField0_ & 0x00000004) == 0x00000004)) {
+            props_ = java.util.Collections.unmodifiableList(props_);
+            bitField0_ = (bitField0_ & ~0x00000004);
+          }
+          result.props_ = props_;
+        } else {
+          result.props_ = propsBuilder_.build();
+        }
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.getDefaultInstance()) return this;
+        if (other.hasExecuteTime()) {
+          setExecuteTime(other.getExecuteTime());
+        }
+        if (other.hasTransactionId()) {
+          setTransactionId(other.getTransactionId());
+        }
+        if (propsBuilder_ == null) {
+          if (!other.props_.isEmpty()) {
+            if (props_.isEmpty()) {
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000004);
+            } else {
+              ensurePropsIsMutable();
+              props_.addAll(other.props_);
+            }
+            onChanged();
+          }
+        } else {
+          if (!other.props_.isEmpty()) {
+            if (propsBuilder_.isEmpty()) {
+              propsBuilder_.dispose();
+              propsBuilder_ = null;
+              props_ = other.props_;
+              bitField0_ = (bitField0_ & ~0x00000004);
+              propsBuilder_ = 
+                com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+                   getPropsFieldBuilder() : null;
+            } else {
+              propsBuilder_.addAllMessages(other.props_);
+            }
+          }
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              executeTime_ = input.readInt64();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              transactionId_ = input.readBytes();
+              break;
+            }
+            case 26: {
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder subBuilder = com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder();
+              input.readMessage(subBuilder, extensionRegistry);
+              addProps(subBuilder.buildPartial());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int64 executeTime = 1;
+      private long executeTime_ ;
+      public boolean hasExecuteTime() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public long getExecuteTime() {
+        return executeTime_;
+      }
+      public Builder setExecuteTime(long value) {
+        bitField0_ |= 0x00000001;
+        executeTime_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearExecuteTime() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        executeTime_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional string transactionId = 2;
+      private java.lang.Object transactionId_ = "";
+      public boolean hasTransactionId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getTransactionId() {
+        java.lang.Object ref = transactionId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          transactionId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setTransactionId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        transactionId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTransactionId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        transactionId_ = getDefaultInstance().getTransactionId();
+        onChanged();
+        return this;
+      }
+      void setTransactionId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        transactionId_ = value;
+        onChanged();
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Pair props = 3;
+      private java.util.List props_ =
+        java.util.Collections.emptyList();
+      private void ensurePropsIsMutable() {
+        if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+          props_ = new java.util.ArrayList(props_);
+          bitField0_ |= 0x00000004;
+         }
+      }
+      
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> propsBuilder_;
+      
+      public java.util.List getPropsList() {
+        if (propsBuilder_ == null) {
+          return java.util.Collections.unmodifiableList(props_);
+        } else {
+          return propsBuilder_.getMessageList();
+        }
+      }
+      public int getPropsCount() {
+        if (propsBuilder_ == null) {
+          return props_.size();
+        } else {
+          return propsBuilder_.getCount();
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getProps(int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);
+        } else {
+          return propsBuilder_.getMessage(index);
+        }
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.set(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, value);
+        }
+        return this;
+      }
+      public Builder setProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.set(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.setMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair value) {
+        if (propsBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          ensurePropsIsMutable();
+          props_.add(index, value);
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, value);
+        }
+        return this;
+      }
+      public Builder addProps(
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addProps(
+          int index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder builderForValue) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.add(index, builderForValue.build());
+          onChanged();
+        } else {
+          propsBuilder_.addMessage(index, builderForValue.build());
+        }
+        return this;
+      }
+      public Builder addAllProps(
+          java.lang.Iterable values) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          super.addAll(values, props_);
+          onChanged();
+        } else {
+          propsBuilder_.addAllMessages(values);
+        }
+        return this;
+      }
+      public Builder clearProps() {
+        if (propsBuilder_ == null) {
+          props_ = java.util.Collections.emptyList();
+          bitField0_ = (bitField0_ & ~0x00000004);
+          onChanged();
+        } else {
+          propsBuilder_.clear();
+        }
+        return this;
+      }
+      public Builder removeProps(int index) {
+        if (propsBuilder_ == null) {
+          ensurePropsIsMutable();
+          props_.remove(index);
+          onChanged();
+        } else {
+          propsBuilder_.remove(index);
+        }
+        return this;
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder getPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().getBuilder(index);
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder getPropsOrBuilder(
+          int index) {
+        if (propsBuilder_ == null) {
+          return props_.get(index);  } else {
+          return propsBuilder_.getMessageOrBuilder(index);
+        }
+      }
+      public java.util.List 
+           getPropsOrBuilderList() {
+        if (propsBuilder_ != null) {
+          return propsBuilder_.getMessageOrBuilderList();
+        } else {
+          return java.util.Collections.unmodifiableList(props_);
+        }
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder() {
+        return getPropsFieldBuilder().addBuilder(
+            com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder addPropsBuilder(
+          int index) {
+        return getPropsFieldBuilder().addBuilder(
+            index, com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance());
+      }
+      public java.util.List 
+           getPropsBuilderList() {
+        return getPropsFieldBuilder().getBuilderList();
+      }
+      private com.google.protobuf.RepeatedFieldBuilder<
+          com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder> 
+          getPropsFieldBuilder() {
+        if (propsBuilder_ == null) {
+          propsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair, com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder, com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder>(
+                  props_,
+                  ((bitField0_ & 0x00000004) == 0x00000004),
+                  getParentForChildren(),
+                  isClean());
+          props_ = null;
+        }
+        return propsBuilder_;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.TransactionEnd)
+    }
+    
+    static {
+      defaultInstance = new TransactionEnd(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.TransactionEnd)
+  }
+  
+  public interface PairOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string key = 1;
+    boolean hasKey();
+    String getKey();
+    
+    // optional string value = 2;
+    boolean hasValue();
+    String getValue();
+  }
+  public static final class Pair extends
+      com.google.protobuf.GeneratedMessage
+      implements PairOrBuilder {
+    // Use Pair.newBuilder() to construct.
+    private Pair(Builder builder) {
+      super(builder);
+    }
+    private Pair(boolean noInit) {}
+    
+    private static final Pair defaultInstance;
+    public static Pair getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Pair getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string key = 1;
+    public static final int KEY_FIELD_NUMBER = 1;
+    private java.lang.Object key_;
+    public boolean hasKey() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getKey() {
+      java.lang.Object ref = key_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          key_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getKeyBytes() {
+      java.lang.Object ref = key_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        key_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string value = 2;
+    public static final int VALUE_FIELD_NUMBER = 2;
+    private java.lang.Object value_;
+    public boolean hasValue() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getValue() {
+      java.lang.Object ref = value_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          value_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getValueBytes() {
+      java.lang.Object ref = value_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        value_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    private void initFields() {
+      key_ = "";
+      value_ = "";
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getKeyBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getValueBytes());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getKeyBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getValueBytes());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalEntry.Pair parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalEntry.Pair prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalEntry.PairOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalEntry.Pair.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        key_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        value_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair build() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Pair result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalEntry.Pair buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalEntry.Pair result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalEntry.Pair buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalEntry.Pair result = new com.alibaba.otter.canal.protocol.CanalEntry.Pair(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.key_ = key_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.value_ = value_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalEntry.Pair) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalEntry.Pair)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalEntry.Pair other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalEntry.Pair.getDefaultInstance()) return this;
+        if (other.hasKey()) {
+          setKey(other.getKey());
+        }
+        if (other.hasValue()) {
+          setValue(other.getValue());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              key_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              value_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string key = 1;
+      private java.lang.Object key_ = "";
+      public boolean hasKey() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getKey() {
+        java.lang.Object ref = key_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          key_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setKey(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        key_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearKey() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        key_ = getDefaultInstance().getKey();
+        onChanged();
+        return this;
+      }
+      void setKey(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        key_ = value;
+        onChanged();
+      }
+      
+      // optional string value = 2;
+      private java.lang.Object value_ = "";
+      public boolean hasValue() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getValue() {
+        java.lang.Object ref = value_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          value_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setValue(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        value_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearValue() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        value_ = getDefaultInstance().getValue();
+        onChanged();
+        return this;
+      }
+      void setValue(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        value_ = value;
+        onChanged();
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Pair)
+    }
+    
+    static {
+      defaultInstance = new Pair(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Pair)
+  }
+  
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Header_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Column_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable;
+  
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\023EntryProtocol.proto\022 com.alibaba.otter" +
+      ".canal.protocol\"\236\001\n\005Entry\0228\n\006header\030\001 \001(" +
+      "\0132(.com.alibaba.otter.canal.protocol.Hea" +
+      "der\022G\n\tentryType\030\002 \001(\0162+.com.alibaba.ott" +
+      "er.canal.protocol.EntryType:\007ROWDATA\022\022\n\n" +
+      "storeValue\030\003 \001(\014\"\203\003\n\006Header\022\022\n\007version\030\001" +
+      " \001(\005:\0011\022\023\n\013logfileName\030\002 \001(\t\022\025\n\rlogfileO" +
+      "ffset\030\003 \001(\003\022\020\n\010serverId\030\004 \001(\003\022\024\n\014servere" +
+      "nCode\030\005 \001(\t\022\023\n\013executeTime\030\006 \001(\003\022A\n\nsour" +
+      "ceType\030\007 \001(\0162&.com.alibaba.otter.canal.p",
+      "rotocol.Type:\005MYSQL\022\022\n\nschemaName\030\010 \001(\t\022" +
+      "\021\n\ttableName\030\t \001(\t\022\023\n\013eventLength\030\n \001(\003\022" +
+      "F\n\teventType\030\013 \001(\0162+.com.alibaba.otter.c" +
+      "anal.protocol.EventType:\006UPDATE\0225\n\005props" +
+      "\030\014 \003(\0132&.com.alibaba.otter.canal.protoco" +
+      "l.Pair\"\326\001\n\006Column\022\r\n\005index\030\001 \001(\005\022\017\n\007sqlT" +
+      "ype\030\002 \001(\005\022\014\n\004name\030\003 \001(\t\022\r\n\005isKey\030\004 \001(\010\022\017" +
+      "\n\007updated\030\005 \001(\010\022\025\n\006isNull\030\006 \001(\010:\005false\0225" +
+      "\n\005props\030\007 \003(\0132&.com.alibaba.otter.canal." +
+      "protocol.Pair\022\r\n\005value\030\010 \001(\t\022\016\n\006length\030\t",
+      " \001(\005\022\021\n\tmysqlType\030\n \001(\t\"\301\001\n\007RowData\022?\n\rb" +
+      "eforeColumns\030\001 \003(\0132(.com.alibaba.otter.c" +
+      "anal.protocol.Column\022>\n\014afterColumns\030\002 \003" +
+      "(\0132(.com.alibaba.otter.canal.protocol.Co" +
+      "lumn\0225\n\005props\030\003 \003(\0132&.com.alibaba.otter." +
+      "canal.protocol.Pair\"\222\002\n\tRowChange\022\017\n\007tab" +
+      "leId\030\001 \001(\003\022F\n\teventType\030\002 \001(\0162+.com.alib" +
+      "aba.otter.canal.protocol.EventType:\006UPDA" +
+      "TE\022\024\n\005isDdl\030\n \001(\010:\005false\022\013\n\003sql\030\013 \001(\t\022;\n" +
+      "\010rowDatas\030\014 \003(\0132).com.alibaba.otter.cana",
+      "l.protocol.RowData\0225\n\005props\030\r \003(\0132&.com." +
+      "alibaba.otter.canal.protocol.Pair\022\025\n\rddl" +
+      "SchemaName\030\016 \001(\t\"\207\001\n\020TransactionBegin\022\023\n" +
+      "\013executeTime\030\001 \001(\003\022\025\n\rtransactionId\030\002 \001(" +
+      "\t\0225\n\005props\030\003 \003(\0132&.com.alibaba.otter.can" +
+      "al.protocol.Pair\022\020\n\010threadId\030\004 \001(\003\"s\n\016Tr" +
+      "ansactionEnd\022\023\n\013executeTime\030\001 \001(\003\022\025\n\rtra" +
+      "nsactionId\030\002 \001(\t\0225\n\005props\030\003 \003(\0132&.com.al" +
+      "ibaba.otter.canal.protocol.Pair\"\"\n\004Pair\022" +
+      "\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t*Q\n\tEntryType",
+      "\022\024\n\020TRANSACTIONBEGIN\020\001\022\013\n\007ROWDATA\020\002\022\022\n\016T" +
+      "RANSACTIONEND\020\003\022\r\n\tHEARTBEAT\020\004*\216\001\n\tEvent" +
+      "Type\022\n\n\006INSERT\020\001\022\n\n\006UPDATE\020\002\022\n\n\006DELETE\020\003" +
+      "\022\n\n\006CREATE\020\004\022\t\n\005ALTER\020\005\022\t\n\005ERASE\020\006\022\t\n\005QU" +
+      "ERY\020\007\022\014\n\010TRUNCATE\020\010\022\n\n\006RENAME\020\t\022\n\n\006CINDE" +
+      "X\020\n\022\n\n\006DINDEX\020\013*(\n\004Type\022\n\n\006ORACLE\020\001\022\t\n\005M" +
+      "YSQL\020\002\022\t\n\005PGSQL\020\003B0\n com.alibaba.otter.c" +
+      "anal.protocolB\nCanalEntryH\001"
+    };
+    com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+      new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+        public com.google.protobuf.ExtensionRegistry assignDescriptors(
+            com.google.protobuf.Descriptors.FileDescriptor root) {
+          descriptor = root;
+          internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor =
+            getDescriptor().getMessageTypes().get(0);
+          internal_static_com_alibaba_otter_canal_protocol_Entry_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Entry_descriptor,
+              new java.lang.String[] { "Header", "EntryType", "StoreValue", },
+              com.alibaba.otter.canal.protocol.CanalEntry.Entry.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.Entry.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Header_descriptor =
+            getDescriptor().getMessageTypes().get(1);
+          internal_static_com_alibaba_otter_canal_protocol_Header_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Header_descriptor,
+              new java.lang.String[] { "Version", "LogfileName", "LogfileOffset", "ServerId", "ServerenCode", "ExecuteTime", "SourceType", "SchemaName", "TableName", "EventLength", "EventType", "Props", },
+              com.alibaba.otter.canal.protocol.CanalEntry.Header.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.Header.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Column_descriptor =
+            getDescriptor().getMessageTypes().get(2);
+          internal_static_com_alibaba_otter_canal_protocol_Column_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Column_descriptor,
+              new java.lang.String[] { "Index", "SqlType", "Name", "IsKey", "Updated", "IsNull", "Props", "Value", "Length", "MysqlType", },
+              com.alibaba.otter.canal.protocol.CanalEntry.Column.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.Column.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor =
+            getDescriptor().getMessageTypes().get(3);
+          internal_static_com_alibaba_otter_canal_protocol_RowData_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_RowData_descriptor,
+              new java.lang.String[] { "BeforeColumns", "AfterColumns", "Props", },
+              com.alibaba.otter.canal.protocol.CanalEntry.RowData.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.RowData.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor =
+            getDescriptor().getMessageTypes().get(4);
+          internal_static_com_alibaba_otter_canal_protocol_RowChange_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_RowChange_descriptor,
+              new java.lang.String[] { "TableId", "EventType", "IsDdl", "Sql", "RowDatas", "Props", "DdlSchemaName", },
+              com.alibaba.otter.canal.protocol.CanalEntry.RowChange.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.RowChange.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor =
+            getDescriptor().getMessageTypes().get(5);
+          internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_TransactionBegin_descriptor,
+              new java.lang.String[] { "ExecuteTime", "TransactionId", "Props", "ThreadId", },
+              com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.TransactionBegin.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor =
+            getDescriptor().getMessageTypes().get(6);
+          internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_TransactionEnd_descriptor,
+              new java.lang.String[] { "ExecuteTime", "TransactionId", "Props", },
+              com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.TransactionEnd.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor =
+            getDescriptor().getMessageTypes().get(7);
+          internal_static_com_alibaba_otter_canal_protocol_Pair_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Pair_descriptor,
+              new java.lang.String[] { "Key", "Value", },
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.class,
+              com.alibaba.otter.canal.protocol.CanalEntry.Pair.Builder.class);
+          return null;
+        }
+      };
+    com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        }, assigner);
+  }
+  
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java
new file mode 100755
index 00000000..890a0930
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java
@@ -0,0 +1,7073 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: CanalProtocol.proto
+
+package com.alibaba.otter.canal.protocol;
+
+public final class CanalPacket {
+  private CanalPacket() {}
+  public static void registerAllExtensions(
+      com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public enum Compression
+      implements com.google.protobuf.ProtocolMessageEnum {
+    NONE(0, 1),
+    ZLIB(1, 2),
+    GZIP(2, 3),
+    LZF(3, 4),
+    ;
+    
+    public static final int NONE_VALUE = 1;
+    public static final int ZLIB_VALUE = 2;
+    public static final int GZIP_VALUE = 3;
+    public static final int LZF_VALUE = 4;
+    
+    
+    public final int getNumber() { return value; }
+    
+    public static Compression valueOf(int value) {
+      switch (value) {
+        case 1: return NONE;
+        case 2: return ZLIB;
+        case 3: return GZIP;
+        case 4: return LZF;
+        default: return null;
+      }
+    }
+    
+    public static com.google.protobuf.Internal.EnumLiteMap
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public Compression findValueByNumber(int number) {
+              return Compression.valueOf(number);
+            }
+          };
+    
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.getDescriptor().getEnumTypes().get(0);
+    }
+    
+    private static final Compression[] VALUES = {
+      NONE, ZLIB, GZIP, LZF, 
+    };
+    
+    public static Compression valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+    
+    private final int index;
+    private final int value;
+    
+    private Compression(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+    
+    // @@protoc_insertion_point(enum_scope:com.alibaba.otter.canal.protocol.Compression)
+  }
+  
+  public enum PacketType
+      implements com.google.protobuf.ProtocolMessageEnum {
+    HANDSHAKE(0, 1),
+    CLIENTAUTHENTICATION(1, 2),
+    ACK(2, 3),
+    SUBSCRIPTION(3, 4),
+    UNSUBSCRIPTION(4, 5),
+    GET(5, 6),
+    MESSAGES(6, 7),
+    CLIENTACK(7, 8),
+    SHUTDOWN(8, 9),
+    DUMP(9, 10),
+    HEARTBEAT(10, 11),
+    CLIENTROLLBACK(11, 12),
+    ;
+    
+    public static final int HANDSHAKE_VALUE = 1;
+    public static final int CLIENTAUTHENTICATION_VALUE = 2;
+    public static final int ACK_VALUE = 3;
+    public static final int SUBSCRIPTION_VALUE = 4;
+    public static final int UNSUBSCRIPTION_VALUE = 5;
+    public static final int GET_VALUE = 6;
+    public static final int MESSAGES_VALUE = 7;
+    public static final int CLIENTACK_VALUE = 8;
+    public static final int SHUTDOWN_VALUE = 9;
+    public static final int DUMP_VALUE = 10;
+    public static final int HEARTBEAT_VALUE = 11;
+    public static final int CLIENTROLLBACK_VALUE = 12;
+    
+    
+    public final int getNumber() { return value; }
+    
+    public static PacketType valueOf(int value) {
+      switch (value) {
+        case 1: return HANDSHAKE;
+        case 2: return CLIENTAUTHENTICATION;
+        case 3: return ACK;
+        case 4: return SUBSCRIPTION;
+        case 5: return UNSUBSCRIPTION;
+        case 6: return GET;
+        case 7: return MESSAGES;
+        case 8: return CLIENTACK;
+        case 9: return SHUTDOWN;
+        case 10: return DUMP;
+        case 11: return HEARTBEAT;
+        case 12: return CLIENTROLLBACK;
+        default: return null;
+      }
+    }
+    
+    public static com.google.protobuf.Internal.EnumLiteMap
+        internalGetValueMap() {
+      return internalValueMap;
+    }
+    private static com.google.protobuf.Internal.EnumLiteMap
+        internalValueMap =
+          new com.google.protobuf.Internal.EnumLiteMap() {
+            public PacketType findValueByNumber(int number) {
+              return PacketType.valueOf(number);
+            }
+          };
+    
+    public final com.google.protobuf.Descriptors.EnumValueDescriptor
+        getValueDescriptor() {
+      return getDescriptor().getValues().get(index);
+    }
+    public final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptorForType() {
+      return getDescriptor();
+    }
+    public static final com.google.protobuf.Descriptors.EnumDescriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.getDescriptor().getEnumTypes().get(1);
+    }
+    
+    private static final PacketType[] VALUES = {
+      HANDSHAKE, CLIENTAUTHENTICATION, ACK, SUBSCRIPTION, UNSUBSCRIPTION, GET, MESSAGES, CLIENTACK, SHUTDOWN, DUMP, HEARTBEAT, CLIENTROLLBACK, 
+    };
+    
+    public static PacketType valueOf(
+        com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+      if (desc.getType() != getDescriptor()) {
+        throw new java.lang.IllegalArgumentException(
+          "EnumValueDescriptor is not for this type.");
+      }
+      return VALUES[desc.getIndex()];
+    }
+    
+    private final int index;
+    private final int value;
+    
+    private PacketType(int index, int value) {
+      this.index = index;
+      this.value = value;
+    }
+    
+    // @@protoc_insertion_point(enum_scope:com.alibaba.otter.canal.protocol.PacketType)
+  }
+  
+  public interface PacketOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int32 magic_number = 1 [default = 17];
+    boolean hasMagicNumber();
+    int getMagicNumber();
+    
+    // optional int32 version = 2 [default = 1];
+    boolean hasVersion();
+    int getVersion();
+    
+    // optional .com.alibaba.otter.canal.protocol.PacketType type = 3;
+    boolean hasType();
+    com.alibaba.otter.canal.protocol.CanalPacket.PacketType getType();
+    
+    // optional .com.alibaba.otter.canal.protocol.Compression compression = 4 [default = NONE];
+    boolean hasCompression();
+    com.alibaba.otter.canal.protocol.CanalPacket.Compression getCompression();
+    
+    // optional bytes body = 5;
+    boolean hasBody();
+    com.google.protobuf.ByteString getBody();
+  }
+  public static final class Packet extends
+      com.google.protobuf.GeneratedMessage
+      implements PacketOrBuilder {
+    // Use Packet.newBuilder() to construct.
+    private Packet(Builder builder) {
+      super(builder);
+    }
+    private Packet(boolean noInit) {}
+    
+    private static final Packet defaultInstance;
+    public static Packet getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Packet getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Packet_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Packet_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int32 magic_number = 1 [default = 17];
+    public static final int MAGIC_NUMBER_FIELD_NUMBER = 1;
+    private int magicNumber_;
+    public boolean hasMagicNumber() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public int getMagicNumber() {
+      return magicNumber_;
+    }
+    
+    // optional int32 version = 2 [default = 1];
+    public static final int VERSION_FIELD_NUMBER = 2;
+    private int version_;
+    public boolean hasVersion() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public int getVersion() {
+      return version_;
+    }
+    
+    // optional .com.alibaba.otter.canal.protocol.PacketType type = 3;
+    public static final int TYPE_FIELD_NUMBER = 3;
+    private com.alibaba.otter.canal.protocol.CanalPacket.PacketType type_;
+    public boolean hasType() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public com.alibaba.otter.canal.protocol.CanalPacket.PacketType getType() {
+      return type_;
+    }
+    
+    // optional .com.alibaba.otter.canal.protocol.Compression compression = 4 [default = NONE];
+    public static final int COMPRESSION_FIELD_NUMBER = 4;
+    private com.alibaba.otter.canal.protocol.CanalPacket.Compression compression_;
+    public boolean hasCompression() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public com.alibaba.otter.canal.protocol.CanalPacket.Compression getCompression() {
+      return compression_;
+    }
+    
+    // optional bytes body = 5;
+    public static final int BODY_FIELD_NUMBER = 5;
+    private com.google.protobuf.ByteString body_;
+    public boolean hasBody() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public com.google.protobuf.ByteString getBody() {
+      return body_;
+    }
+    
+    private void initFields() {
+      magicNumber_ = 17;
+      version_ = 1;
+      type_ = com.alibaba.otter.canal.protocol.CanalPacket.PacketType.HANDSHAKE;
+      compression_ = com.alibaba.otter.canal.protocol.CanalPacket.Compression.NONE;
+      body_ = com.google.protobuf.ByteString.EMPTY;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(1, magicNumber_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeInt32(2, version_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeEnum(3, type_.getNumber());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeEnum(4, compression_.getNumber());
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBytes(5, body_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(1, magicNumber_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(2, version_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(3, type_.getNumber());
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeEnumSize(4, compression_.getNumber());
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(5, body_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Packet parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Packet prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.PacketOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Packet_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Packet_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Packet.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        magicNumber_ = 17;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        version_ = 1;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        type_ = com.alibaba.otter.canal.protocol.CanalPacket.PacketType.HANDSHAKE;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        compression_ = com.alibaba.otter.canal.protocol.CanalPacket.Compression.NONE;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        body_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000010);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Packet.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Packet getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Packet.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Packet build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Packet result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Packet buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Packet result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Packet buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Packet result = new com.alibaba.otter.canal.protocol.CanalPacket.Packet(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.magicNumber_ = magicNumber_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.version_ = version_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.type_ = type_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.compression_ = compression_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.body_ = body_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Packet) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Packet)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Packet other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Packet.getDefaultInstance()) return this;
+        if (other.hasMagicNumber()) {
+          setMagicNumber(other.getMagicNumber());
+        }
+        if (other.hasVersion()) {
+          setVersion(other.getVersion());
+        }
+        if (other.hasType()) {
+          setType(other.getType());
+        }
+        if (other.hasCompression()) {
+          setCompression(other.getCompression());
+        }
+        if (other.hasBody()) {
+          setBody(other.getBody());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              magicNumber_ = input.readInt32();
+              break;
+            }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              version_ = input.readInt32();
+              break;
+            }
+            case 24: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalPacket.PacketType value = com.alibaba.otter.canal.protocol.CanalPacket.PacketType.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(3, rawValue);
+              } else {
+                bitField0_ |= 0x00000004;
+                type_ = value;
+              }
+              break;
+            }
+            case 32: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalPacket.Compression value = com.alibaba.otter.canal.protocol.CanalPacket.Compression.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(4, rawValue);
+              } else {
+                bitField0_ |= 0x00000008;
+                compression_ = value;
+              }
+              break;
+            }
+            case 42: {
+              bitField0_ |= 0x00000010;
+              body_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int32 magic_number = 1 [default = 17];
+      private int magicNumber_ = 17;
+      public boolean hasMagicNumber() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public int getMagicNumber() {
+        return magicNumber_;
+      }
+      public Builder setMagicNumber(int value) {
+        bitField0_ |= 0x00000001;
+        magicNumber_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearMagicNumber() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        magicNumber_ = 17;
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 version = 2 [default = 1];
+      private int version_ = 1;
+      public boolean hasVersion() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public int getVersion() {
+        return version_;
+      }
+      public Builder setVersion(int value) {
+        bitField0_ |= 0x00000002;
+        version_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearVersion() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        version_ = 1;
+        onChanged();
+        return this;
+      }
+      
+      // optional .com.alibaba.otter.canal.protocol.PacketType type = 3;
+      private com.alibaba.otter.canal.protocol.CanalPacket.PacketType type_ = com.alibaba.otter.canal.protocol.CanalPacket.PacketType.HANDSHAKE;
+      public boolean hasType() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public com.alibaba.otter.canal.protocol.CanalPacket.PacketType getType() {
+        return type_;
+      }
+      public Builder setType(com.alibaba.otter.canal.protocol.CanalPacket.PacketType value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000004;
+        type_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearType() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        type_ = com.alibaba.otter.canal.protocol.CanalPacket.PacketType.HANDSHAKE;
+        onChanged();
+        return this;
+      }
+      
+      // optional .com.alibaba.otter.canal.protocol.Compression compression = 4 [default = NONE];
+      private com.alibaba.otter.canal.protocol.CanalPacket.Compression compression_ = com.alibaba.otter.canal.protocol.CanalPacket.Compression.NONE;
+      public boolean hasCompression() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public com.alibaba.otter.canal.protocol.CanalPacket.Compression getCompression() {
+        return compression_;
+      }
+      public Builder setCompression(com.alibaba.otter.canal.protocol.CanalPacket.Compression value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        bitField0_ |= 0x00000008;
+        compression_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearCompression() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        compression_ = com.alibaba.otter.canal.protocol.CanalPacket.Compression.NONE;
+        onChanged();
+        return this;
+      }
+      
+      // optional bytes body = 5;
+      private com.google.protobuf.ByteString body_ = com.google.protobuf.ByteString.EMPTY;
+      public boolean hasBody() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public com.google.protobuf.ByteString getBody() {
+        return body_;
+      }
+      public Builder setBody(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000010;
+        body_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearBody() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        body_ = getDefaultInstance().getBody();
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Packet)
+    }
+    
+    static {
+      defaultInstance = new Packet(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Packet)
+  }
+  
+  public interface HeartBeatOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int64 send_timestamp = 1;
+    boolean hasSendTimestamp();
+    long getSendTimestamp();
+    
+    // optional int64 start_timestamp = 2;
+    boolean hasStartTimestamp();
+    long getStartTimestamp();
+  }
+  public static final class HeartBeat extends
+      com.google.protobuf.GeneratedMessage
+      implements HeartBeatOrBuilder {
+    // Use HeartBeat.newBuilder() to construct.
+    private HeartBeat(Builder builder) {
+      super(builder);
+    }
+    private HeartBeat(boolean noInit) {}
+    
+    private static final HeartBeat defaultInstance;
+    public static HeartBeat getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public HeartBeat getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_HeartBeat_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_HeartBeat_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int64 send_timestamp = 1;
+    public static final int SEND_TIMESTAMP_FIELD_NUMBER = 1;
+    private long sendTimestamp_;
+    public boolean hasSendTimestamp() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public long getSendTimestamp() {
+      return sendTimestamp_;
+    }
+    
+    // optional int64 start_timestamp = 2;
+    public static final int START_TIMESTAMP_FIELD_NUMBER = 2;
+    private long startTimestamp_;
+    public boolean hasStartTimestamp() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public long getStartTimestamp() {
+      return startTimestamp_;
+    }
+    
+    private void initFields() {
+      sendTimestamp_ = 0L;
+      startTimestamp_ = 0L;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt64(1, sendTimestamp_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeInt64(2, startTimestamp_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(1, sendTimestamp_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(2, startTimestamp_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.HeartBeatOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_HeartBeat_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_HeartBeat_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        sendTimestamp_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        startTimestamp_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat result = new com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.sendTimestamp_ = sendTimestamp_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.startTimestamp_ = startTimestamp_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat.getDefaultInstance()) return this;
+        if (other.hasSendTimestamp()) {
+          setSendTimestamp(other.getSendTimestamp());
+        }
+        if (other.hasStartTimestamp()) {
+          setStartTimestamp(other.getStartTimestamp());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              sendTimestamp_ = input.readInt64();
+              break;
+            }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              startTimestamp_ = input.readInt64();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int64 send_timestamp = 1;
+      private long sendTimestamp_ ;
+      public boolean hasSendTimestamp() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public long getSendTimestamp() {
+        return sendTimestamp_;
+      }
+      public Builder setSendTimestamp(long value) {
+        bitField0_ |= 0x00000001;
+        sendTimestamp_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSendTimestamp() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        sendTimestamp_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional int64 start_timestamp = 2;
+      private long startTimestamp_ ;
+      public boolean hasStartTimestamp() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public long getStartTimestamp() {
+        return startTimestamp_;
+      }
+      public Builder setStartTimestamp(long value) {
+        bitField0_ |= 0x00000002;
+        startTimestamp_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearStartTimestamp() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        startTimestamp_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.HeartBeat)
+    }
+    
+    static {
+      defaultInstance = new HeartBeat(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.HeartBeat)
+  }
+  
+  public interface HandshakeOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string communication_encoding = 1 [default = "utf8"];
+    boolean hasCommunicationEncoding();
+    String getCommunicationEncoding();
+    
+    // optional bytes seeds = 2;
+    boolean hasSeeds();
+    com.google.protobuf.ByteString getSeeds();
+    
+    // repeated .com.alibaba.otter.canal.protocol.Compression supported_compressions = 3;
+    java.util.List getSupportedCompressionsList();
+    int getSupportedCompressionsCount();
+    com.alibaba.otter.canal.protocol.CanalPacket.Compression getSupportedCompressions(int index);
+  }
+  public static final class Handshake extends
+      com.google.protobuf.GeneratedMessage
+      implements HandshakeOrBuilder {
+    // Use Handshake.newBuilder() to construct.
+    private Handshake(Builder builder) {
+      super(builder);
+    }
+    private Handshake(boolean noInit) {}
+    
+    private static final Handshake defaultInstance;
+    public static Handshake getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Handshake getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Handshake_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Handshake_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string communication_encoding = 1 [default = "utf8"];
+    public static final int COMMUNICATION_ENCODING_FIELD_NUMBER = 1;
+    private java.lang.Object communicationEncoding_;
+    public boolean hasCommunicationEncoding() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getCommunicationEncoding() {
+      java.lang.Object ref = communicationEncoding_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          communicationEncoding_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getCommunicationEncodingBytes() {
+      java.lang.Object ref = communicationEncoding_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        communicationEncoding_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional bytes seeds = 2;
+    public static final int SEEDS_FIELD_NUMBER = 2;
+    private com.google.protobuf.ByteString seeds_;
+    public boolean hasSeeds() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public com.google.protobuf.ByteString getSeeds() {
+      return seeds_;
+    }
+    
+    // repeated .com.alibaba.otter.canal.protocol.Compression supported_compressions = 3;
+    public static final int SUPPORTED_COMPRESSIONS_FIELD_NUMBER = 3;
+    private java.util.List supportedCompressions_;
+    public java.util.List getSupportedCompressionsList() {
+      return supportedCompressions_;
+    }
+    public int getSupportedCompressionsCount() {
+      return supportedCompressions_.size();
+    }
+    public com.alibaba.otter.canal.protocol.CanalPacket.Compression getSupportedCompressions(int index) {
+      return supportedCompressions_.get(index);
+    }
+    
+    private void initFields() {
+      communicationEncoding_ = "utf8";
+      seeds_ = com.google.protobuf.ByteString.EMPTY;
+      supportedCompressions_ = java.util.Collections.emptyList();
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getCommunicationEncodingBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, seeds_);
+      }
+      for (int i = 0; i < supportedCompressions_.size(); i++) {
+        output.writeEnum(3, supportedCompressions_.get(i).getNumber());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getCommunicationEncodingBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, seeds_);
+      }
+      {
+        int dataSize = 0;
+        for (int i = 0; i < supportedCompressions_.size(); i++) {
+          dataSize += com.google.protobuf.CodedOutputStream
+            .computeEnumSizeNoTag(supportedCompressions_.get(i).getNumber());
+        }
+        size += dataSize;
+        size += 1 * supportedCompressions_.size();
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Handshake parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Handshake prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.HandshakeOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Handshake_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Handshake_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Handshake.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        communicationEncoding_ = "utf8";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        seeds_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        supportedCompressions_ = java.util.Collections.emptyList();
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Handshake.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Handshake getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Handshake.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Handshake build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Handshake result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Handshake buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Handshake result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Handshake buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Handshake result = new com.alibaba.otter.canal.protocol.CanalPacket.Handshake(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.communicationEncoding_ = communicationEncoding_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.seeds_ = seeds_;
+        if (((bitField0_ & 0x00000004) == 0x00000004)) {
+          supportedCompressions_ = java.util.Collections.unmodifiableList(supportedCompressions_);
+          bitField0_ = (bitField0_ & ~0x00000004);
+        }
+        result.supportedCompressions_ = supportedCompressions_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Handshake) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Handshake)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Handshake other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Handshake.getDefaultInstance()) return this;
+        if (other.hasCommunicationEncoding()) {
+          setCommunicationEncoding(other.getCommunicationEncoding());
+        }
+        if (other.hasSeeds()) {
+          setSeeds(other.getSeeds());
+        }
+        if (!other.supportedCompressions_.isEmpty()) {
+          if (supportedCompressions_.isEmpty()) {
+            supportedCompressions_ = other.supportedCompressions_;
+            bitField0_ = (bitField0_ & ~0x00000004);
+          } else {
+            ensureSupportedCompressionsIsMutable();
+            supportedCompressions_.addAll(other.supportedCompressions_);
+          }
+          onChanged();
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              communicationEncoding_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              seeds_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              int rawValue = input.readEnum();
+              com.alibaba.otter.canal.protocol.CanalPacket.Compression value = com.alibaba.otter.canal.protocol.CanalPacket.Compression.valueOf(rawValue);
+              if (value == null) {
+                unknownFields.mergeVarintField(3, rawValue);
+              } else {
+                addSupportedCompressions(value);
+              }
+              break;
+            }
+            case 26: {
+              int length = input.readRawVarint32();
+              int oldLimit = input.pushLimit(length);
+              while(input.getBytesUntilLimit() > 0) {
+                int rawValue = input.readEnum();
+                com.alibaba.otter.canal.protocol.CanalPacket.Compression value = com.alibaba.otter.canal.protocol.CanalPacket.Compression.valueOf(rawValue);
+                if (value == null) {
+                  unknownFields.mergeVarintField(3, rawValue);
+                } else {
+                  addSupportedCompressions(value);
+                }
+              }
+              input.popLimit(oldLimit);
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string communication_encoding = 1 [default = "utf8"];
+      private java.lang.Object communicationEncoding_ = "utf8";
+      public boolean hasCommunicationEncoding() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getCommunicationEncoding() {
+        java.lang.Object ref = communicationEncoding_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          communicationEncoding_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setCommunicationEncoding(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        communicationEncoding_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearCommunicationEncoding() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        communicationEncoding_ = getDefaultInstance().getCommunicationEncoding();
+        onChanged();
+        return this;
+      }
+      void setCommunicationEncoding(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        communicationEncoding_ = value;
+        onChanged();
+      }
+      
+      // optional bytes seeds = 2;
+      private com.google.protobuf.ByteString seeds_ = com.google.protobuf.ByteString.EMPTY;
+      public boolean hasSeeds() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public com.google.protobuf.ByteString getSeeds() {
+        return seeds_;
+      }
+      public Builder setSeeds(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        seeds_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearSeeds() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        seeds_ = getDefaultInstance().getSeeds();
+        onChanged();
+        return this;
+      }
+      
+      // repeated .com.alibaba.otter.canal.protocol.Compression supported_compressions = 3;
+      private java.util.List supportedCompressions_ =
+        java.util.Collections.emptyList();
+      private void ensureSupportedCompressionsIsMutable() {
+        if (!((bitField0_ & 0x00000004) == 0x00000004)) {
+          supportedCompressions_ = new java.util.ArrayList(supportedCompressions_);
+          bitField0_ |= 0x00000004;
+        }
+      }
+      public java.util.List getSupportedCompressionsList() {
+        return java.util.Collections.unmodifiableList(supportedCompressions_);
+      }
+      public int getSupportedCompressionsCount() {
+        return supportedCompressions_.size();
+      }
+      public com.alibaba.otter.canal.protocol.CanalPacket.Compression getSupportedCompressions(int index) {
+        return supportedCompressions_.get(index);
+      }
+      public Builder setSupportedCompressions(
+          int index, com.alibaba.otter.canal.protocol.CanalPacket.Compression value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        ensureSupportedCompressionsIsMutable();
+        supportedCompressions_.set(index, value);
+        onChanged();
+        return this;
+      }
+      public Builder addSupportedCompressions(com.alibaba.otter.canal.protocol.CanalPacket.Compression value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
+        ensureSupportedCompressionsIsMutable();
+        supportedCompressions_.add(value);
+        onChanged();
+        return this;
+      }
+      public Builder addAllSupportedCompressions(
+          java.lang.Iterable values) {
+        ensureSupportedCompressionsIsMutable();
+        super.addAll(values, supportedCompressions_);
+        onChanged();
+        return this;
+      }
+      public Builder clearSupportedCompressions() {
+        supportedCompressions_ = java.util.Collections.emptyList();
+        bitField0_ = (bitField0_ & ~0x00000004);
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Handshake)
+    }
+    
+    static {
+      defaultInstance = new Handshake(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Handshake)
+  }
+  
+  public interface ClientAuthOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string username = 1;
+    boolean hasUsername();
+    String getUsername();
+    
+    // optional bytes password = 2;
+    boolean hasPassword();
+    com.google.protobuf.ByteString getPassword();
+    
+    // optional int32 net_read_timeout = 3 [default = 0];
+    boolean hasNetReadTimeout();
+    int getNetReadTimeout();
+    
+    // optional int32 net_write_timeout = 4 [default = 0];
+    boolean hasNetWriteTimeout();
+    int getNetWriteTimeout();
+    
+    // optional string destination = 5;
+    boolean hasDestination();
+    String getDestination();
+    
+    // optional string client_id = 6;
+    boolean hasClientId();
+    String getClientId();
+    
+    // optional string filter = 7;
+    boolean hasFilter();
+    String getFilter();
+    
+    // optional int64 start_timestamp = 8;
+    boolean hasStartTimestamp();
+    long getStartTimestamp();
+  }
+  public static final class ClientAuth extends
+      com.google.protobuf.GeneratedMessage
+      implements ClientAuthOrBuilder {
+    // Use ClientAuth.newBuilder() to construct.
+    private ClientAuth(Builder builder) {
+      super(builder);
+    }
+    private ClientAuth(boolean noInit) {}
+    
+    private static final ClientAuth defaultInstance;
+    public static ClientAuth getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public ClientAuth getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAuth_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAuth_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string username = 1;
+    public static final int USERNAME_FIELD_NUMBER = 1;
+    private java.lang.Object username_;
+    public boolean hasUsername() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getUsername() {
+      java.lang.Object ref = username_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          username_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getUsernameBytes() {
+      java.lang.Object ref = username_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        username_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional bytes password = 2;
+    public static final int PASSWORD_FIELD_NUMBER = 2;
+    private com.google.protobuf.ByteString password_;
+    public boolean hasPassword() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public com.google.protobuf.ByteString getPassword() {
+      return password_;
+    }
+    
+    // optional int32 net_read_timeout = 3 [default = 0];
+    public static final int NET_READ_TIMEOUT_FIELD_NUMBER = 3;
+    private int netReadTimeout_;
+    public boolean hasNetReadTimeout() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public int getNetReadTimeout() {
+      return netReadTimeout_;
+    }
+    
+    // optional int32 net_write_timeout = 4 [default = 0];
+    public static final int NET_WRITE_TIMEOUT_FIELD_NUMBER = 4;
+    private int netWriteTimeout_;
+    public boolean hasNetWriteTimeout() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public int getNetWriteTimeout() {
+      return netWriteTimeout_;
+    }
+    
+    // optional string destination = 5;
+    public static final int DESTINATION_FIELD_NUMBER = 5;
+    private java.lang.Object destination_;
+    public boolean hasDestination() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public String getDestination() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          destination_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDestinationBytes() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        destination_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string client_id = 6;
+    public static final int CLIENT_ID_FIELD_NUMBER = 6;
+    private java.lang.Object clientId_;
+    public boolean hasClientId() {
+      return ((bitField0_ & 0x00000020) == 0x00000020);
+    }
+    public String getClientId() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          clientId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClientIdBytes() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        clientId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string filter = 7;
+    public static final int FILTER_FIELD_NUMBER = 7;
+    private java.lang.Object filter_;
+    public boolean hasFilter() {
+      return ((bitField0_ & 0x00000040) == 0x00000040);
+    }
+    public String getFilter() {
+      java.lang.Object ref = filter_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          filter_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getFilterBytes() {
+      java.lang.Object ref = filter_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        filter_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 start_timestamp = 8;
+    public static final int START_TIMESTAMP_FIELD_NUMBER = 8;
+    private long startTimestamp_;
+    public boolean hasStartTimestamp() {
+      return ((bitField0_ & 0x00000080) == 0x00000080);
+    }
+    public long getStartTimestamp() {
+      return startTimestamp_;
+    }
+    
+    private void initFields() {
+      username_ = "";
+      password_ = com.google.protobuf.ByteString.EMPTY;
+      netReadTimeout_ = 0;
+      netWriteTimeout_ = 0;
+      destination_ = "";
+      clientId_ = "";
+      filter_ = "";
+      startTimestamp_ = 0L;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getUsernameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, password_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt32(3, netReadTimeout_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeInt32(4, netWriteTimeout_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeBytes(5, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        output.writeBytes(6, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        output.writeBytes(7, getFilterBytes());
+      }
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        output.writeInt64(8, startTimestamp_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getUsernameBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, password_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(3, netReadTimeout_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(4, netWriteTimeout_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(5, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(6, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000040) == 0x00000040)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(7, getFilterBytes());
+      }
+      if (((bitField0_ & 0x00000080) == 0x00000080)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(8, startTimestamp_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.ClientAuthOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAuth_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAuth_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        username_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        password_ = com.google.protobuf.ByteString.EMPTY;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        netReadTimeout_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        netWriteTimeout_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        destination_ = "";
+        bitField0_ = (bitField0_ & ~0x00000010);
+        clientId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000020);
+        filter_ = "";
+        bitField0_ = (bitField0_ & ~0x00000040);
+        startTimestamp_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000080);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth result = new com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.username_ = username_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.password_ = password_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.netReadTimeout_ = netReadTimeout_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.netWriteTimeout_ = netWriteTimeout_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.destination_ = destination_;
+        if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+          to_bitField0_ |= 0x00000020;
+        }
+        result.clientId_ = clientId_;
+        if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+          to_bitField0_ |= 0x00000040;
+        }
+        result.filter_ = filter_;
+        if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+          to_bitField0_ |= 0x00000080;
+        }
+        result.startTimestamp_ = startTimestamp_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth.getDefaultInstance()) return this;
+        if (other.hasUsername()) {
+          setUsername(other.getUsername());
+        }
+        if (other.hasPassword()) {
+          setPassword(other.getPassword());
+        }
+        if (other.hasNetReadTimeout()) {
+          setNetReadTimeout(other.getNetReadTimeout());
+        }
+        if (other.hasNetWriteTimeout()) {
+          setNetWriteTimeout(other.getNetWriteTimeout());
+        }
+        if (other.hasDestination()) {
+          setDestination(other.getDestination());
+        }
+        if (other.hasClientId()) {
+          setClientId(other.getClientId());
+        }
+        if (other.hasFilter()) {
+          setFilter(other.getFilter());
+        }
+        if (other.hasStartTimestamp()) {
+          setStartTimestamp(other.getStartTimestamp());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              username_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              password_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              netReadTimeout_ = input.readInt32();
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              netWriteTimeout_ = input.readInt32();
+              break;
+            }
+            case 42: {
+              bitField0_ |= 0x00000010;
+              destination_ = input.readBytes();
+              break;
+            }
+            case 50: {
+              bitField0_ |= 0x00000020;
+              clientId_ = input.readBytes();
+              break;
+            }
+            case 58: {
+              bitField0_ |= 0x00000040;
+              filter_ = input.readBytes();
+              break;
+            }
+            case 64: {
+              bitField0_ |= 0x00000080;
+              startTimestamp_ = input.readInt64();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string username = 1;
+      private java.lang.Object username_ = "";
+      public boolean hasUsername() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getUsername() {
+        java.lang.Object ref = username_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          username_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setUsername(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        username_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearUsername() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        username_ = getDefaultInstance().getUsername();
+        onChanged();
+        return this;
+      }
+      void setUsername(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        username_ = value;
+        onChanged();
+      }
+      
+      // optional bytes password = 2;
+      private com.google.protobuf.ByteString password_ = com.google.protobuf.ByteString.EMPTY;
+      public boolean hasPassword() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public com.google.protobuf.ByteString getPassword() {
+        return password_;
+      }
+      public Builder setPassword(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        password_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearPassword() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        password_ = getDefaultInstance().getPassword();
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 net_read_timeout = 3 [default = 0];
+      private int netReadTimeout_ ;
+      public boolean hasNetReadTimeout() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public int getNetReadTimeout() {
+        return netReadTimeout_;
+      }
+      public Builder setNetReadTimeout(int value) {
+        bitField0_ |= 0x00000004;
+        netReadTimeout_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearNetReadTimeout() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        netReadTimeout_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 net_write_timeout = 4 [default = 0];
+      private int netWriteTimeout_ ;
+      public boolean hasNetWriteTimeout() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public int getNetWriteTimeout() {
+        return netWriteTimeout_;
+      }
+      public Builder setNetWriteTimeout(int value) {
+        bitField0_ |= 0x00000008;
+        netWriteTimeout_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearNetWriteTimeout() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        netWriteTimeout_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional string destination = 5;
+      private java.lang.Object destination_ = "";
+      public boolean hasDestination() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public String getDestination() {
+        java.lang.Object ref = destination_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          destination_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDestination(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000010;
+        destination_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDestination() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        destination_ = getDefaultInstance().getDestination();
+        onChanged();
+        return this;
+      }
+      void setDestination(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000010;
+        destination_ = value;
+        onChanged();
+      }
+      
+      // optional string client_id = 6;
+      private java.lang.Object clientId_ = "";
+      public boolean hasClientId() {
+        return ((bitField0_ & 0x00000020) == 0x00000020);
+      }
+      public String getClientId() {
+        java.lang.Object ref = clientId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          clientId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClientId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000020;
+        clientId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClientId() {
+        bitField0_ = (bitField0_ & ~0x00000020);
+        clientId_ = getDefaultInstance().getClientId();
+        onChanged();
+        return this;
+      }
+      void setClientId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000020;
+        clientId_ = value;
+        onChanged();
+      }
+      
+      // optional string filter = 7;
+      private java.lang.Object filter_ = "";
+      public boolean hasFilter() {
+        return ((bitField0_ & 0x00000040) == 0x00000040);
+      }
+      public String getFilter() {
+        java.lang.Object ref = filter_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          filter_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setFilter(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000040;
+        filter_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFilter() {
+        bitField0_ = (bitField0_ & ~0x00000040);
+        filter_ = getDefaultInstance().getFilter();
+        onChanged();
+        return this;
+      }
+      void setFilter(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000040;
+        filter_ = value;
+        onChanged();
+      }
+      
+      // optional int64 start_timestamp = 8;
+      private long startTimestamp_ ;
+      public boolean hasStartTimestamp() {
+        return ((bitField0_ & 0x00000080) == 0x00000080);
+      }
+      public long getStartTimestamp() {
+        return startTimestamp_;
+      }
+      public Builder setStartTimestamp(long value) {
+        bitField0_ |= 0x00000080;
+        startTimestamp_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearStartTimestamp() {
+        bitField0_ = (bitField0_ & ~0x00000080);
+        startTimestamp_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.ClientAuth)
+    }
+    
+    static {
+      defaultInstance = new ClientAuth(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.ClientAuth)
+  }
+  
+  public interface AckOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int32 error_code = 1 [default = 0];
+    boolean hasErrorCode();
+    int getErrorCode();
+    
+    // optional string error_message = 2;
+    boolean hasErrorMessage();
+    String getErrorMessage();
+  }
+  public static final class Ack extends
+      com.google.protobuf.GeneratedMessage
+      implements AckOrBuilder {
+    // Use Ack.newBuilder() to construct.
+    private Ack(Builder builder) {
+      super(builder);
+    }
+    private Ack(boolean noInit) {}
+    
+    private static final Ack defaultInstance;
+    public static Ack getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Ack getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Ack_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Ack_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int32 error_code = 1 [default = 0];
+    public static final int ERROR_CODE_FIELD_NUMBER = 1;
+    private int errorCode_;
+    public boolean hasErrorCode() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public int getErrorCode() {
+      return errorCode_;
+    }
+    
+    // optional string error_message = 2;
+    public static final int ERROR_MESSAGE_FIELD_NUMBER = 2;
+    private java.lang.Object errorMessage_;
+    public boolean hasErrorMessage() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getErrorMessage() {
+      java.lang.Object ref = errorMessage_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          errorMessage_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getErrorMessageBytes() {
+      java.lang.Object ref = errorMessage_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        errorMessage_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    private void initFields() {
+      errorCode_ = 0;
+      errorMessage_ = "";
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt32(1, errorCode_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getErrorMessageBytes());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(1, errorCode_);
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getErrorMessageBytes());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Ack parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Ack prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.AckOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Ack_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Ack_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Ack.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        errorCode_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        errorMessage_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Ack.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Ack getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Ack.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Ack build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Ack result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Ack buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Ack result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Ack buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Ack result = new com.alibaba.otter.canal.protocol.CanalPacket.Ack(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.errorCode_ = errorCode_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.errorMessage_ = errorMessage_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Ack) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Ack)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Ack other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Ack.getDefaultInstance()) return this;
+        if (other.hasErrorCode()) {
+          setErrorCode(other.getErrorCode());
+        }
+        if (other.hasErrorMessage()) {
+          setErrorMessage(other.getErrorMessage());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              errorCode_ = input.readInt32();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              errorMessage_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int32 error_code = 1 [default = 0];
+      private int errorCode_ ;
+      public boolean hasErrorCode() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public int getErrorCode() {
+        return errorCode_;
+      }
+      public Builder setErrorCode(int value) {
+        bitField0_ |= 0x00000001;
+        errorCode_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearErrorCode() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        errorCode_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional string error_message = 2;
+      private java.lang.Object errorMessage_ = "";
+      public boolean hasErrorMessage() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getErrorMessage() {
+        java.lang.Object ref = errorMessage_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          errorMessage_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setErrorMessage(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        errorMessage_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearErrorMessage() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        errorMessage_ = getDefaultInstance().getErrorMessage();
+        onChanged();
+        return this;
+      }
+      void setErrorMessage(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        errorMessage_ = value;
+        onChanged();
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Ack)
+    }
+    
+    static {
+      defaultInstance = new Ack(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Ack)
+  }
+  
+  public interface ClientAckOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string destination = 1;
+    boolean hasDestination();
+    String getDestination();
+    
+    // optional string client_id = 2;
+    boolean hasClientId();
+    String getClientId();
+    
+    // optional int64 batch_id = 3;
+    boolean hasBatchId();
+    long getBatchId();
+  }
+  public static final class ClientAck extends
+      com.google.protobuf.GeneratedMessage
+      implements ClientAckOrBuilder {
+    // Use ClientAck.newBuilder() to construct.
+    private ClientAck(Builder builder) {
+      super(builder);
+    }
+    private ClientAck(boolean noInit) {}
+    
+    private static final ClientAck defaultInstance;
+    public static ClientAck getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public ClientAck getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAck_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAck_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string destination = 1;
+    public static final int DESTINATION_FIELD_NUMBER = 1;
+    private java.lang.Object destination_;
+    public boolean hasDestination() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDestination() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          destination_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDestinationBytes() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        destination_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string client_id = 2;
+    public static final int CLIENT_ID_FIELD_NUMBER = 2;
+    private java.lang.Object clientId_;
+    public boolean hasClientId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getClientId() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          clientId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClientIdBytes() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        clientId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 batch_id = 3;
+    public static final int BATCH_ID_FIELD_NUMBER = 3;
+    private long batchId_;
+    public boolean hasBatchId() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getBatchId() {
+      return batchId_;
+    }
+    
+    private void initFields() {
+      destination_ = "";
+      clientId_ = "";
+      batchId_ = 0L;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(3, batchId_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(3, batchId_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientAck parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.ClientAck prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.ClientAckOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAck_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientAck_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.ClientAck.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        destination_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        clientId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        batchId_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.ClientAck.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientAck getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.ClientAck.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientAck build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientAck result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.ClientAck buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientAck result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientAck buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientAck result = new com.alibaba.otter.canal.protocol.CanalPacket.ClientAck(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.destination_ = destination_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.clientId_ = clientId_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.batchId_ = batchId_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.ClientAck) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.ClientAck)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.ClientAck other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.ClientAck.getDefaultInstance()) return this;
+        if (other.hasDestination()) {
+          setDestination(other.getDestination());
+        }
+        if (other.hasClientId()) {
+          setClientId(other.getClientId());
+        }
+        if (other.hasBatchId()) {
+          setBatchId(other.getBatchId());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              destination_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              clientId_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              batchId_ = input.readInt64();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string destination = 1;
+      private java.lang.Object destination_ = "";
+      public boolean hasDestination() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDestination() {
+        java.lang.Object ref = destination_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          destination_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDestination(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDestination() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        destination_ = getDefaultInstance().getDestination();
+        onChanged();
+        return this;
+      }
+      void setDestination(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+      }
+      
+      // optional string client_id = 2;
+      private java.lang.Object clientId_ = "";
+      public boolean hasClientId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getClientId() {
+        java.lang.Object ref = clientId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          clientId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClientId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClientId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        clientId_ = getDefaultInstance().getClientId();
+        onChanged();
+        return this;
+      }
+      void setClientId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+      }
+      
+      // optional int64 batch_id = 3;
+      private long batchId_ ;
+      public boolean hasBatchId() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public long getBatchId() {
+        return batchId_;
+      }
+      public Builder setBatchId(long value) {
+        bitField0_ |= 0x00000004;
+        batchId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearBatchId() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        batchId_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.ClientAck)
+    }
+    
+    static {
+      defaultInstance = new ClientAck(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.ClientAck)
+  }
+  
+  public interface SubOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string destination = 1;
+    boolean hasDestination();
+    String getDestination();
+    
+    // optional string client_id = 2;
+    boolean hasClientId();
+    String getClientId();
+    
+    // optional string filter = 7;
+    boolean hasFilter();
+    String getFilter();
+  }
+  public static final class Sub extends
+      com.google.protobuf.GeneratedMessage
+      implements SubOrBuilder {
+    // Use Sub.newBuilder() to construct.
+    private Sub(Builder builder) {
+      super(builder);
+    }
+    private Sub(boolean noInit) {}
+    
+    private static final Sub defaultInstance;
+    public static Sub getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Sub getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Sub_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Sub_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string destination = 1;
+    public static final int DESTINATION_FIELD_NUMBER = 1;
+    private java.lang.Object destination_;
+    public boolean hasDestination() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDestination() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          destination_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDestinationBytes() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        destination_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string client_id = 2;
+    public static final int CLIENT_ID_FIELD_NUMBER = 2;
+    private java.lang.Object clientId_;
+    public boolean hasClientId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getClientId() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          clientId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClientIdBytes() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        clientId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string filter = 7;
+    public static final int FILTER_FIELD_NUMBER = 7;
+    private java.lang.Object filter_;
+    public boolean hasFilter() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public String getFilter() {
+      java.lang.Object ref = filter_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          filter_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getFilterBytes() {
+      java.lang.Object ref = filter_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        filter_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    private void initFields() {
+      destination_ = "";
+      clientId_ = "";
+      filter_ = "";
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(7, getFilterBytes());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(7, getFilterBytes());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Sub parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Sub prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.SubOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Sub_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Sub_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Sub.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        destination_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        clientId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        filter_ = "";
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Sub.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Sub getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Sub.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Sub build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Sub result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Sub buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Sub result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Sub buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Sub result = new com.alibaba.otter.canal.protocol.CanalPacket.Sub(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.destination_ = destination_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.clientId_ = clientId_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.filter_ = filter_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Sub) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Sub)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Sub other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Sub.getDefaultInstance()) return this;
+        if (other.hasDestination()) {
+          setDestination(other.getDestination());
+        }
+        if (other.hasClientId()) {
+          setClientId(other.getClientId());
+        }
+        if (other.hasFilter()) {
+          setFilter(other.getFilter());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              destination_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              clientId_ = input.readBytes();
+              break;
+            }
+            case 58: {
+              bitField0_ |= 0x00000004;
+              filter_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string destination = 1;
+      private java.lang.Object destination_ = "";
+      public boolean hasDestination() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDestination() {
+        java.lang.Object ref = destination_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          destination_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDestination(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDestination() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        destination_ = getDefaultInstance().getDestination();
+        onChanged();
+        return this;
+      }
+      void setDestination(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+      }
+      
+      // optional string client_id = 2;
+      private java.lang.Object clientId_ = "";
+      public boolean hasClientId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getClientId() {
+        java.lang.Object ref = clientId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          clientId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClientId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClientId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        clientId_ = getDefaultInstance().getClientId();
+        onChanged();
+        return this;
+      }
+      void setClientId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+      }
+      
+      // optional string filter = 7;
+      private java.lang.Object filter_ = "";
+      public boolean hasFilter() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public String getFilter() {
+        java.lang.Object ref = filter_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          filter_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setFilter(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        filter_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFilter() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        filter_ = getDefaultInstance().getFilter();
+        onChanged();
+        return this;
+      }
+      void setFilter(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000004;
+        filter_ = value;
+        onChanged();
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Sub)
+    }
+    
+    static {
+      defaultInstance = new Sub(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Sub)
+  }
+  
+  public interface UnsubOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string destination = 1;
+    boolean hasDestination();
+    String getDestination();
+    
+    // optional string client_id = 2;
+    boolean hasClientId();
+    String getClientId();
+    
+    // optional string filter = 7;
+    boolean hasFilter();
+    String getFilter();
+  }
+  public static final class Unsub extends
+      com.google.protobuf.GeneratedMessage
+      implements UnsubOrBuilder {
+    // Use Unsub.newBuilder() to construct.
+    private Unsub(Builder builder) {
+      super(builder);
+    }
+    private Unsub(boolean noInit) {}
+    
+    private static final Unsub defaultInstance;
+    public static Unsub getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Unsub getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Unsub_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Unsub_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string destination = 1;
+    public static final int DESTINATION_FIELD_NUMBER = 1;
+    private java.lang.Object destination_;
+    public boolean hasDestination() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDestination() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          destination_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDestinationBytes() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        destination_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string client_id = 2;
+    public static final int CLIENT_ID_FIELD_NUMBER = 2;
+    private java.lang.Object clientId_;
+    public boolean hasClientId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getClientId() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          clientId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClientIdBytes() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        clientId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string filter = 7;
+    public static final int FILTER_FIELD_NUMBER = 7;
+    private java.lang.Object filter_;
+    public boolean hasFilter() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public String getFilter() {
+      java.lang.Object ref = filter_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          filter_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getFilterBytes() {
+      java.lang.Object ref = filter_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        filter_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    private void initFields() {
+      destination_ = "";
+      clientId_ = "";
+      filter_ = "";
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeBytes(7, getFilterBytes());
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(7, getFilterBytes());
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Unsub parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Unsub prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.UnsubOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Unsub_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Unsub_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Unsub.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        destination_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        clientId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        filter_ = "";
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Unsub.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Unsub getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Unsub.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Unsub build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Unsub result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Unsub buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Unsub result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Unsub buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Unsub result = new com.alibaba.otter.canal.protocol.CanalPacket.Unsub(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.destination_ = destination_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.clientId_ = clientId_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.filter_ = filter_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Unsub) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Unsub)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Unsub other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Unsub.getDefaultInstance()) return this;
+        if (other.hasDestination()) {
+          setDestination(other.getDestination());
+        }
+        if (other.hasClientId()) {
+          setClientId(other.getClientId());
+        }
+        if (other.hasFilter()) {
+          setFilter(other.getFilter());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              destination_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              clientId_ = input.readBytes();
+              break;
+            }
+            case 58: {
+              bitField0_ |= 0x00000004;
+              filter_ = input.readBytes();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string destination = 1;
+      private java.lang.Object destination_ = "";
+      public boolean hasDestination() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDestination() {
+        java.lang.Object ref = destination_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          destination_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDestination(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDestination() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        destination_ = getDefaultInstance().getDestination();
+        onChanged();
+        return this;
+      }
+      void setDestination(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+      }
+      
+      // optional string client_id = 2;
+      private java.lang.Object clientId_ = "";
+      public boolean hasClientId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getClientId() {
+        java.lang.Object ref = clientId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          clientId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClientId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClientId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        clientId_ = getDefaultInstance().getClientId();
+        onChanged();
+        return this;
+      }
+      void setClientId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+      }
+      
+      // optional string filter = 7;
+      private java.lang.Object filter_ = "";
+      public boolean hasFilter() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public String getFilter() {
+        java.lang.Object ref = filter_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          filter_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setFilter(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000004;
+        filter_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFilter() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        filter_ = getDefaultInstance().getFilter();
+        onChanged();
+        return this;
+      }
+      void setFilter(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000004;
+        filter_ = value;
+        onChanged();
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Unsub)
+    }
+    
+    static {
+      defaultInstance = new Unsub(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Unsub)
+  }
+  
+  public interface GetOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string destination = 1;
+    boolean hasDestination();
+    String getDestination();
+    
+    // optional string client_id = 2;
+    boolean hasClientId();
+    String getClientId();
+    
+    // optional int32 fetch_size = 3;
+    boolean hasFetchSize();
+    int getFetchSize();
+    
+    // optional int64 timeout = 4 [default = -1];
+    boolean hasTimeout();
+    long getTimeout();
+    
+    // optional int32 unit = 5 [default = 2];
+    boolean hasUnit();
+    int getUnit();
+    
+    // optional bool auto_ack = 6 [default = false];
+    boolean hasAutoAck();
+    boolean getAutoAck();
+  }
+  public static final class Get extends
+      com.google.protobuf.GeneratedMessage
+      implements GetOrBuilder {
+    // Use Get.newBuilder() to construct.
+    private Get(Builder builder) {
+      super(builder);
+    }
+    private Get(boolean noInit) {}
+    
+    private static final Get defaultInstance;
+    public static Get getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Get getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Get_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Get_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string destination = 1;
+    public static final int DESTINATION_FIELD_NUMBER = 1;
+    private java.lang.Object destination_;
+    public boolean hasDestination() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDestination() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          destination_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDestinationBytes() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        destination_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string client_id = 2;
+    public static final int CLIENT_ID_FIELD_NUMBER = 2;
+    private java.lang.Object clientId_;
+    public boolean hasClientId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getClientId() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          clientId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClientIdBytes() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        clientId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int32 fetch_size = 3;
+    public static final int FETCH_SIZE_FIELD_NUMBER = 3;
+    private int fetchSize_;
+    public boolean hasFetchSize() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public int getFetchSize() {
+      return fetchSize_;
+    }
+    
+    // optional int64 timeout = 4 [default = -1];
+    public static final int TIMEOUT_FIELD_NUMBER = 4;
+    private long timeout_;
+    public boolean hasTimeout() {
+      return ((bitField0_ & 0x00000008) == 0x00000008);
+    }
+    public long getTimeout() {
+      return timeout_;
+    }
+    
+    // optional int32 unit = 5 [default = 2];
+    public static final int UNIT_FIELD_NUMBER = 5;
+    private int unit_;
+    public boolean hasUnit() {
+      return ((bitField0_ & 0x00000010) == 0x00000010);
+    }
+    public int getUnit() {
+      return unit_;
+    }
+    
+    // optional bool auto_ack = 6 [default = false];
+    public static final int AUTO_ACK_FIELD_NUMBER = 6;
+    private boolean autoAck_;
+    public boolean hasAutoAck() {
+      return ((bitField0_ & 0x00000020) == 0x00000020);
+    }
+    public boolean getAutoAck() {
+      return autoAck_;
+    }
+    
+    private void initFields() {
+      destination_ = "";
+      clientId_ = "";
+      fetchSize_ = 0;
+      timeout_ = -1L;
+      unit_ = 2;
+      autoAck_ = false;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt32(3, fetchSize_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        output.writeInt64(4, timeout_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        output.writeInt32(5, unit_);
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        output.writeBool(6, autoAck_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(3, fetchSize_);
+      }
+      if (((bitField0_ & 0x00000008) == 0x00000008)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(4, timeout_);
+      }
+      if (((bitField0_ & 0x00000010) == 0x00000010)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(5, unit_);
+      }
+      if (((bitField0_ & 0x00000020) == 0x00000020)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBoolSize(6, autoAck_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Get parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Get prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.GetOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Get_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Get_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Get.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        destination_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        clientId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        fetchSize_ = 0;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        timeout_ = -1L;
+        bitField0_ = (bitField0_ & ~0x00000008);
+        unit_ = 2;
+        bitField0_ = (bitField0_ & ~0x00000010);
+        autoAck_ = false;
+        bitField0_ = (bitField0_ & ~0x00000020);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Get.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Get getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Get.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Get build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Get result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Get buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Get result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Get buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Get result = new com.alibaba.otter.canal.protocol.CanalPacket.Get(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.destination_ = destination_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.clientId_ = clientId_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.fetchSize_ = fetchSize_;
+        if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+          to_bitField0_ |= 0x00000008;
+        }
+        result.timeout_ = timeout_;
+        if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+          to_bitField0_ |= 0x00000010;
+        }
+        result.unit_ = unit_;
+        if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+          to_bitField0_ |= 0x00000020;
+        }
+        result.autoAck_ = autoAck_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Get) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Get)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Get other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Get.getDefaultInstance()) return this;
+        if (other.hasDestination()) {
+          setDestination(other.getDestination());
+        }
+        if (other.hasClientId()) {
+          setClientId(other.getClientId());
+        }
+        if (other.hasFetchSize()) {
+          setFetchSize(other.getFetchSize());
+        }
+        if (other.hasTimeout()) {
+          setTimeout(other.getTimeout());
+        }
+        if (other.hasUnit()) {
+          setUnit(other.getUnit());
+        }
+        if (other.hasAutoAck()) {
+          setAutoAck(other.getAutoAck());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              destination_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              clientId_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              fetchSize_ = input.readInt32();
+              break;
+            }
+            case 32: {
+              bitField0_ |= 0x00000008;
+              timeout_ = input.readInt64();
+              break;
+            }
+            case 40: {
+              bitField0_ |= 0x00000010;
+              unit_ = input.readInt32();
+              break;
+            }
+            case 48: {
+              bitField0_ |= 0x00000020;
+              autoAck_ = input.readBool();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string destination = 1;
+      private java.lang.Object destination_ = "";
+      public boolean hasDestination() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDestination() {
+        java.lang.Object ref = destination_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          destination_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDestination(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDestination() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        destination_ = getDefaultInstance().getDestination();
+        onChanged();
+        return this;
+      }
+      void setDestination(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+      }
+      
+      // optional string client_id = 2;
+      private java.lang.Object clientId_ = "";
+      public boolean hasClientId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getClientId() {
+        java.lang.Object ref = clientId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          clientId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClientId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClientId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        clientId_ = getDefaultInstance().getClientId();
+        onChanged();
+        return this;
+      }
+      void setClientId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+      }
+      
+      // optional int32 fetch_size = 3;
+      private int fetchSize_ ;
+      public boolean hasFetchSize() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public int getFetchSize() {
+        return fetchSize_;
+      }
+      public Builder setFetchSize(int value) {
+        bitField0_ |= 0x00000004;
+        fetchSize_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearFetchSize() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        fetchSize_ = 0;
+        onChanged();
+        return this;
+      }
+      
+      // optional int64 timeout = 4 [default = -1];
+      private long timeout_ = -1L;
+      public boolean hasTimeout() {
+        return ((bitField0_ & 0x00000008) == 0x00000008);
+      }
+      public long getTimeout() {
+        return timeout_;
+      }
+      public Builder setTimeout(long value) {
+        bitField0_ |= 0x00000008;
+        timeout_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTimeout() {
+        bitField0_ = (bitField0_ & ~0x00000008);
+        timeout_ = -1L;
+        onChanged();
+        return this;
+      }
+      
+      // optional int32 unit = 5 [default = 2];
+      private int unit_ = 2;
+      public boolean hasUnit() {
+        return ((bitField0_ & 0x00000010) == 0x00000010);
+      }
+      public int getUnit() {
+        return unit_;
+      }
+      public Builder setUnit(int value) {
+        bitField0_ |= 0x00000010;
+        unit_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearUnit() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        unit_ = 2;
+        onChanged();
+        return this;
+      }
+      
+      // optional bool auto_ack = 6 [default = false];
+      private boolean autoAck_ ;
+      public boolean hasAutoAck() {
+        return ((bitField0_ & 0x00000020) == 0x00000020);
+      }
+      public boolean getAutoAck() {
+        return autoAck_;
+      }
+      public Builder setAutoAck(boolean value) {
+        bitField0_ |= 0x00000020;
+        autoAck_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearAutoAck() {
+        bitField0_ = (bitField0_ & ~0x00000020);
+        autoAck_ = false;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Get)
+    }
+    
+    static {
+      defaultInstance = new Get(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Get)
+  }
+  
+  public interface MessagesOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional int64 batch_id = 1;
+    boolean hasBatchId();
+    long getBatchId();
+    
+    // repeated bytes messages = 2;
+    java.util.List getMessagesList();
+    int getMessagesCount();
+    com.google.protobuf.ByteString getMessages(int index);
+  }
+  public static final class Messages extends
+      com.google.protobuf.GeneratedMessage
+      implements MessagesOrBuilder {
+    // Use Messages.newBuilder() to construct.
+    private Messages(Builder builder) {
+      super(builder);
+    }
+    private Messages(boolean noInit) {}
+    
+    private static final Messages defaultInstance;
+    public static Messages getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Messages getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Messages_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Messages_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional int64 batch_id = 1;
+    public static final int BATCH_ID_FIELD_NUMBER = 1;
+    private long batchId_;
+    public boolean hasBatchId() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public long getBatchId() {
+      return batchId_;
+    }
+    
+    // repeated bytes messages = 2;
+    public static final int MESSAGES_FIELD_NUMBER = 2;
+    private java.util.List messages_;
+    public java.util.List
+        getMessagesList() {
+      return messages_;
+    }
+    public int getMessagesCount() {
+      return messages_.size();
+    }
+    public com.google.protobuf.ByteString getMessages(int index) {
+      return messages_.get(index);
+    }
+    
+    private void initFields() {
+      batchId_ = 0L;
+      messages_ = java.util.Collections.emptyList();;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeInt64(1, batchId_);
+      }
+      for (int i = 0; i < messages_.size(); i++) {
+        output.writeBytes(2, messages_.get(i));
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(1, batchId_);
+      }
+      {
+        int dataSize = 0;
+        for (int i = 0; i < messages_.size(); i++) {
+          dataSize += com.google.protobuf.CodedOutputStream
+            .computeBytesSizeNoTag(messages_.get(i));
+        }
+        size += dataSize;
+        size += 1 * getMessagesList().size();
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Messages parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Messages prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.MessagesOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Messages_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Messages_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Messages.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        batchId_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000001);
+        messages_ = java.util.Collections.emptyList();;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Messages.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Messages getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Messages.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Messages build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Messages result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Messages buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Messages result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Messages buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Messages result = new com.alibaba.otter.canal.protocol.CanalPacket.Messages(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.batchId_ = batchId_;
+        if (((bitField0_ & 0x00000002) == 0x00000002)) {
+          messages_ = java.util.Collections.unmodifiableList(messages_);
+          bitField0_ = (bitField0_ & ~0x00000002);
+        }
+        result.messages_ = messages_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Messages) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Messages)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Messages other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Messages.getDefaultInstance()) return this;
+        if (other.hasBatchId()) {
+          setBatchId(other.getBatchId());
+        }
+        if (!other.messages_.isEmpty()) {
+          if (messages_.isEmpty()) {
+            messages_ = other.messages_;
+            bitField0_ = (bitField0_ & ~0x00000002);
+          } else {
+            ensureMessagesIsMutable();
+            messages_.addAll(other.messages_);
+          }
+          onChanged();
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 8: {
+              bitField0_ |= 0x00000001;
+              batchId_ = input.readInt64();
+              break;
+            }
+            case 18: {
+              ensureMessagesIsMutable();
+              messages_.add(input.readBytes());
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional int64 batch_id = 1;
+      private long batchId_ ;
+      public boolean hasBatchId() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public long getBatchId() {
+        return batchId_;
+      }
+      public Builder setBatchId(long value) {
+        bitField0_ |= 0x00000001;
+        batchId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearBatchId() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        batchId_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // repeated bytes messages = 2;
+      private java.util.List messages_ = java.util.Collections.emptyList();;
+      private void ensureMessagesIsMutable() {
+        if (!((bitField0_ & 0x00000002) == 0x00000002)) {
+          messages_ = new java.util.ArrayList(messages_);
+          bitField0_ |= 0x00000002;
+         }
+      }
+      public java.util.List
+          getMessagesList() {
+        return java.util.Collections.unmodifiableList(messages_);
+      }
+      public int getMessagesCount() {
+        return messages_.size();
+      }
+      public com.google.protobuf.ByteString getMessages(int index) {
+        return messages_.get(index);
+      }
+      public Builder setMessages(
+          int index, com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  ensureMessagesIsMutable();
+        messages_.set(index, value);
+        onChanged();
+        return this;
+      }
+      public Builder addMessages(com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  ensureMessagesIsMutable();
+        messages_.add(value);
+        onChanged();
+        return this;
+      }
+      public Builder addAllMessages(
+          java.lang.Iterable values) {
+        ensureMessagesIsMutable();
+        super.addAll(values, messages_);
+        onChanged();
+        return this;
+      }
+      public Builder clearMessages() {
+        messages_ = java.util.Collections.emptyList();;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Messages)
+    }
+    
+    static {
+      defaultInstance = new Messages(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Messages)
+  }
+  
+  public interface DumpOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string journal = 1;
+    boolean hasJournal();
+    String getJournal();
+    
+    // optional int64 position = 2;
+    boolean hasPosition();
+    long getPosition();
+    
+    // optional int64 timestamp = 3 [default = 0];
+    boolean hasTimestamp();
+    long getTimestamp();
+  }
+  public static final class Dump extends
+      com.google.protobuf.GeneratedMessage
+      implements DumpOrBuilder {
+    // Use Dump.newBuilder() to construct.
+    private Dump(Builder builder) {
+      super(builder);
+    }
+    private Dump(boolean noInit) {}
+    
+    private static final Dump defaultInstance;
+    public static Dump getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public Dump getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Dump_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Dump_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string journal = 1;
+    public static final int JOURNAL_FIELD_NUMBER = 1;
+    private java.lang.Object journal_;
+    public boolean hasJournal() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getJournal() {
+      java.lang.Object ref = journal_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          journal_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getJournalBytes() {
+      java.lang.Object ref = journal_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        journal_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 position = 2;
+    public static final int POSITION_FIELD_NUMBER = 2;
+    private long position_;
+    public boolean hasPosition() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public long getPosition() {
+      return position_;
+    }
+    
+    // optional int64 timestamp = 3 [default = 0];
+    public static final int TIMESTAMP_FIELD_NUMBER = 3;
+    private long timestamp_;
+    public boolean hasTimestamp() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getTimestamp() {
+      return timestamp_;
+    }
+    
+    private void initFields() {
+      journal_ = "";
+      position_ = 0L;
+      timestamp_ = 0L;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getJournalBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeInt64(2, position_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(3, timestamp_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getJournalBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(2, position_);
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(3, timestamp_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.Dump parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.Dump prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.DumpOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Dump_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_Dump_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.Dump.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        journal_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        position_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000002);
+        timestamp_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Dump.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Dump getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.Dump.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Dump build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Dump result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.Dump buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.Dump result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.Dump buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.Dump result = new com.alibaba.otter.canal.protocol.CanalPacket.Dump(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.journal_ = journal_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.position_ = position_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.timestamp_ = timestamp_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.Dump) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.Dump)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.Dump other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.Dump.getDefaultInstance()) return this;
+        if (other.hasJournal()) {
+          setJournal(other.getJournal());
+        }
+        if (other.hasPosition()) {
+          setPosition(other.getPosition());
+        }
+        if (other.hasTimestamp()) {
+          setTimestamp(other.getTimestamp());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              journal_ = input.readBytes();
+              break;
+            }
+            case 16: {
+              bitField0_ |= 0x00000002;
+              position_ = input.readInt64();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              timestamp_ = input.readInt64();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string journal = 1;
+      private java.lang.Object journal_ = "";
+      public boolean hasJournal() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getJournal() {
+        java.lang.Object ref = journal_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          journal_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setJournal(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        journal_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearJournal() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        journal_ = getDefaultInstance().getJournal();
+        onChanged();
+        return this;
+      }
+      void setJournal(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        journal_ = value;
+        onChanged();
+      }
+      
+      // optional int64 position = 2;
+      private long position_ ;
+      public boolean hasPosition() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public long getPosition() {
+        return position_;
+      }
+      public Builder setPosition(long value) {
+        bitField0_ |= 0x00000002;
+        position_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearPosition() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        position_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // optional int64 timestamp = 3 [default = 0];
+      private long timestamp_ ;
+      public boolean hasTimestamp() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public long getTimestamp() {
+        return timestamp_;
+      }
+      public Builder setTimestamp(long value) {
+        bitField0_ |= 0x00000004;
+        timestamp_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearTimestamp() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        timestamp_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.Dump)
+    }
+    
+    static {
+      defaultInstance = new Dump(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.Dump)
+  }
+  
+  public interface ClientRollbackOrBuilder
+      extends com.google.protobuf.MessageOrBuilder {
+    
+    // optional string destination = 1;
+    boolean hasDestination();
+    String getDestination();
+    
+    // optional string client_id = 2;
+    boolean hasClientId();
+    String getClientId();
+    
+    // optional int64 batch_id = 3;
+    boolean hasBatchId();
+    long getBatchId();
+  }
+  public static final class ClientRollback extends
+      com.google.protobuf.GeneratedMessage
+      implements ClientRollbackOrBuilder {
+    // Use ClientRollback.newBuilder() to construct.
+    private ClientRollback(Builder builder) {
+      super(builder);
+    }
+    private ClientRollback(boolean noInit) {}
+    
+    private static final ClientRollback defaultInstance;
+    public static ClientRollback getDefaultInstance() {
+      return defaultInstance;
+    }
+    
+    public ClientRollback getDefaultInstanceForType() {
+      return defaultInstance;
+    }
+    
+    public static final com.google.protobuf.Descriptors.Descriptor
+        getDescriptor() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientRollback_descriptor;
+    }
+    
+    protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+        internalGetFieldAccessorTable() {
+      return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientRollback_fieldAccessorTable;
+    }
+    
+    private int bitField0_;
+    // optional string destination = 1;
+    public static final int DESTINATION_FIELD_NUMBER = 1;
+    private java.lang.Object destination_;
+    public boolean hasDestination() {
+      return ((bitField0_ & 0x00000001) == 0x00000001);
+    }
+    public String getDestination() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          destination_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getDestinationBytes() {
+      java.lang.Object ref = destination_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        destination_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional string client_id = 2;
+    public static final int CLIENT_ID_FIELD_NUMBER = 2;
+    private java.lang.Object clientId_;
+    public boolean hasClientId() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public String getClientId() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        return (String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        String s = bs.toStringUtf8();
+        if (com.google.protobuf.Internal.isValidUtf8(bs)) {
+          clientId_ = s;
+        }
+        return s;
+      }
+    }
+    private com.google.protobuf.ByteString getClientIdBytes() {
+      java.lang.Object ref = clientId_;
+      if (ref instanceof String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+        clientId_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+    
+    // optional int64 batch_id = 3;
+    public static final int BATCH_ID_FIELD_NUMBER = 3;
+    private long batchId_;
+    public boolean hasBatchId() {
+      return ((bitField0_ & 0x00000004) == 0x00000004);
+    }
+    public long getBatchId() {
+      return batchId_;
+    }
+    
+    private void initFields() {
+      destination_ = "";
+      clientId_ = "";
+      batchId_ = 0L;
+    }
+    private byte memoizedIsInitialized = -1;
+    public final boolean isInitialized() {
+      byte isInitialized = memoizedIsInitialized;
+      if (isInitialized != -1) return isInitialized == 1;
+      
+      memoizedIsInitialized = 1;
+      return true;
+    }
+    
+    public void writeTo(com.google.protobuf.CodedOutputStream output)
+                        throws java.io.IOException {
+      getSerializedSize();
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        output.writeBytes(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeBytes(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        output.writeInt64(3, batchId_);
+      }
+      getUnknownFields().writeTo(output);
+    }
+    
+    private int memoizedSerializedSize = -1;
+    public int getSerializedSize() {
+      int size = memoizedSerializedSize;
+      if (size != -1) return size;
+    
+      size = 0;
+      if (((bitField0_ & 0x00000001) == 0x00000001)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(1, getDestinationBytes());
+      }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeBytesSize(2, getClientIdBytes());
+      }
+      if (((bitField0_ & 0x00000004) == 0x00000004)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt64Size(3, batchId_);
+      }
+      size += getUnknownFields().getSerializedSize();
+      memoizedSerializedSize = size;
+      return size;
+    }
+    
+    private static final long serialVersionUID = 0L;
+    @java.lang.Override
+    protected java.lang.Object writeReplace()
+        throws java.io.ObjectStreamException {
+      return super.writeReplace();
+    }
+    
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(
+        com.google.protobuf.ByteString data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(
+        com.google.protobuf.ByteString data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(byte[] data)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(
+        byte[] data,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws com.google.protobuf.InvalidProtocolBufferException {
+      return newBuilder().mergeFrom(data, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseDelimitedFrom(java.io.InputStream input)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseDelimitedFrom(
+        java.io.InputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      Builder builder = newBuilder();
+      if (builder.mergeDelimitedFrom(input, extensionRegistry)) {
+        return builder.buildParsed();
+      } else {
+        return null;
+      }
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(
+        com.google.protobuf.CodedInputStream input)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input).buildParsed();
+    }
+    public static com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback parseFrom(
+        com.google.protobuf.CodedInputStream input,
+        com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+        throws java.io.IOException {
+      return newBuilder().mergeFrom(input, extensionRegistry)
+               .buildParsed();
+    }
+    
+    public static Builder newBuilder() { return Builder.create(); }
+    public Builder newBuilderForType() { return newBuilder(); }
+    public static Builder newBuilder(com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback prototype) {
+      return newBuilder().mergeFrom(prototype);
+    }
+    public Builder toBuilder() { return newBuilder(this); }
+    
+    @java.lang.Override
+    protected Builder newBuilderForType(
+        com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+      Builder builder = new Builder(parent);
+      return builder;
+    }
+    public static final class Builder extends
+        com.google.protobuf.GeneratedMessage.Builder
+       implements com.alibaba.otter.canal.protocol.CanalPacket.ClientRollbackOrBuilder {
+      public static final com.google.protobuf.Descriptors.Descriptor
+          getDescriptor() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientRollback_descriptor;
+      }
+      
+      protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+          internalGetFieldAccessorTable() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.internal_static_com_alibaba_otter_canal_protocol_ClientRollback_fieldAccessorTable;
+      }
+      
+      // Construct using com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback.newBuilder()
+      private Builder() {
+        maybeForceBuilderInitialization();
+      }
+      
+      private Builder(BuilderParent parent) {
+        super(parent);
+        maybeForceBuilderInitialization();
+      }
+      private void maybeForceBuilderInitialization() {
+        if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+        }
+      }
+      private static Builder create() {
+        return new Builder();
+      }
+      
+      public Builder clear() {
+        super.clear();
+        destination_ = "";
+        bitField0_ = (bitField0_ & ~0x00000001);
+        clientId_ = "";
+        bitField0_ = (bitField0_ & ~0x00000002);
+        batchId_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000004);
+        return this;
+      }
+      
+      public Builder clone() {
+        return create().mergeFrom(buildPartial());
+      }
+      
+      public com.google.protobuf.Descriptors.Descriptor
+          getDescriptorForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback.getDescriptor();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback getDefaultInstanceForType() {
+        return com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback.getDefaultInstance();
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback build() {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(result);
+        }
+        return result;
+      }
+      
+      private com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback buildParsed()
+          throws com.google.protobuf.InvalidProtocolBufferException {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback result = buildPartial();
+        if (!result.isInitialized()) {
+          throw newUninitializedMessageException(
+            result).asInvalidProtocolBufferException();
+        }
+        return result;
+      }
+      
+      public com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback buildPartial() {
+        com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback result = new com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback(this);
+        int from_bitField0_ = bitField0_;
+        int to_bitField0_ = 0;
+        if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+          to_bitField0_ |= 0x00000001;
+        }
+        result.destination_ = destination_;
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.clientId_ = clientId_;
+        if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+          to_bitField0_ |= 0x00000004;
+        }
+        result.batchId_ = batchId_;
+        result.bitField0_ = to_bitField0_;
+        onBuilt();
+        return result;
+      }
+      
+      public Builder mergeFrom(com.google.protobuf.Message other) {
+        if (other instanceof com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback) {
+          return mergeFrom((com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback)other);
+        } else {
+          super.mergeFrom(other);
+          return this;
+        }
+      }
+      
+      public Builder mergeFrom(com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback other) {
+        if (other == com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback.getDefaultInstance()) return this;
+        if (other.hasDestination()) {
+          setDestination(other.getDestination());
+        }
+        if (other.hasClientId()) {
+          setClientId(other.getClientId());
+        }
+        if (other.hasBatchId()) {
+          setBatchId(other.getBatchId());
+        }
+        this.mergeUnknownFields(other.getUnknownFields());
+        return this;
+      }
+      
+      public final boolean isInitialized() {
+        return true;
+      }
+      
+      public Builder mergeFrom(
+          com.google.protobuf.CodedInputStream input,
+          com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+          throws java.io.IOException {
+        com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+          com.google.protobuf.UnknownFieldSet.newBuilder(
+            this.getUnknownFields());
+        while (true) {
+          int tag = input.readTag();
+          switch (tag) {
+            case 0:
+              this.setUnknownFields(unknownFields.build());
+              onChanged();
+              return this;
+            default: {
+              if (!parseUnknownField(input, unknownFields,
+                                     extensionRegistry, tag)) {
+                this.setUnknownFields(unknownFields.build());
+                onChanged();
+                return this;
+              }
+              break;
+            }
+            case 10: {
+              bitField0_ |= 0x00000001;
+              destination_ = input.readBytes();
+              break;
+            }
+            case 18: {
+              bitField0_ |= 0x00000002;
+              clientId_ = input.readBytes();
+              break;
+            }
+            case 24: {
+              bitField0_ |= 0x00000004;
+              batchId_ = input.readInt64();
+              break;
+            }
+          }
+        }
+      }
+      
+      private int bitField0_;
+      
+      // optional string destination = 1;
+      private java.lang.Object destination_ = "";
+      public boolean hasDestination() {
+        return ((bitField0_ & 0x00000001) == 0x00000001);
+      }
+      public String getDestination() {
+        java.lang.Object ref = destination_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          destination_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setDestination(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearDestination() {
+        bitField0_ = (bitField0_ & ~0x00000001);
+        destination_ = getDefaultInstance().getDestination();
+        onChanged();
+        return this;
+      }
+      void setDestination(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000001;
+        destination_ = value;
+        onChanged();
+      }
+      
+      // optional string client_id = 2;
+      private java.lang.Object clientId_ = "";
+      public boolean hasClientId() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public String getClientId() {
+        java.lang.Object ref = clientId_;
+        if (!(ref instanceof String)) {
+          String s = ((com.google.protobuf.ByteString) ref).toStringUtf8();
+          clientId_ = s;
+          return s;
+        } else {
+          return (String) ref;
+        }
+      }
+      public Builder setClientId(String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearClientId() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        clientId_ = getDefaultInstance().getClientId();
+        onChanged();
+        return this;
+      }
+      void setClientId(com.google.protobuf.ByteString value) {
+        bitField0_ |= 0x00000002;
+        clientId_ = value;
+        onChanged();
+      }
+      
+      // optional int64 batch_id = 3;
+      private long batchId_ ;
+      public boolean hasBatchId() {
+        return ((bitField0_ & 0x00000004) == 0x00000004);
+      }
+      public long getBatchId() {
+        return batchId_;
+      }
+      public Builder setBatchId(long value) {
+        bitField0_ |= 0x00000004;
+        batchId_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearBatchId() {
+        bitField0_ = (bitField0_ & ~0x00000004);
+        batchId_ = 0L;
+        onChanged();
+        return this;
+      }
+      
+      // @@protoc_insertion_point(builder_scope:com.alibaba.otter.canal.protocol.ClientRollback)
+    }
+    
+    static {
+      defaultInstance = new ClientRollback(true);
+      defaultInstance.initFields();
+    }
+    
+    // @@protoc_insertion_point(class_scope:com.alibaba.otter.canal.protocol.ClientRollback)
+  }
+  
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Packet_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Packet_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_HeartBeat_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_HeartBeat_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Handshake_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Handshake_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_ClientAuth_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_ClientAuth_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Ack_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Ack_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_ClientAck_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_ClientAck_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Sub_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Sub_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Unsub_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Unsub_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Get_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Get_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Messages_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Messages_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_Dump_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_Dump_fieldAccessorTable;
+  private static com.google.protobuf.Descriptors.Descriptor
+    internal_static_com_alibaba_otter_canal_protocol_ClientRollback_descriptor;
+  private static
+    com.google.protobuf.GeneratedMessage.FieldAccessorTable
+      internal_static_com_alibaba_otter_canal_protocol_ClientRollback_fieldAccessorTable;
+  
+  public static com.google.protobuf.Descriptors.FileDescriptor
+      getDescriptor() {
+    return descriptor;
+  }
+  private static com.google.protobuf.Descriptors.FileDescriptor
+      descriptor;
+  static {
+    java.lang.String[] descriptorData = {
+      "\n\023CanalProtocol.proto\022 com.alibaba.otter" +
+      ".canal.protocol\"\312\001\n\006Packet\022\030\n\014magic_numb" +
+      "er\030\001 \001(\005:\00217\022\022\n\007version\030\002 \001(\005:\0011\022:\n\004type" +
+      "\030\003 \001(\0162,.com.alibaba.otter.canal.protoco" +
+      "l.PacketType\022H\n\013compression\030\004 \001(\0162-.com." +
+      "alibaba.otter.canal.protocol.Compression" +
+      ":\004NONE\022\014\n\004body\030\005 \001(\014\"<\n\tHeartBeat\022\026\n\016sen" +
+      "d_timestamp\030\001 \001(\003\022\027\n\017start_timestamp\030\002 \001" +
+      "(\003\"\217\001\n\tHandshake\022$\n\026communication_encodi" +
+      "ng\030\001 \001(\t:\004utf8\022\r\n\005seeds\030\002 \001(\014\022M\n\026support",
+      "ed_compressions\030\003 \003(\0162-.com.alibaba.otte" +
+      "r.canal.protocol.Compression\"\274\001\n\nClientA" +
+      "uth\022\020\n\010username\030\001 \001(\t\022\020\n\010password\030\002 \001(\014\022" +
+      "\033\n\020net_read_timeout\030\003 \001(\005:\0010\022\034\n\021net_writ" +
+      "e_timeout\030\004 \001(\005:\0010\022\023\n\013destination\030\005 \001(\t\022" +
+      "\021\n\tclient_id\030\006 \001(\t\022\016\n\006filter\030\007 \001(\t\022\027\n\017st" +
+      "art_timestamp\030\010 \001(\003\"3\n\003Ack\022\025\n\nerror_code" +
+      "\030\001 \001(\005:\0010\022\025\n\rerror_message\030\002 \001(\t\"E\n\tClie" +
+      "ntAck\022\023\n\013destination\030\001 \001(\t\022\021\n\tclient_id\030" +
+      "\002 \001(\t\022\020\n\010batch_id\030\003 \001(\003\"=\n\003Sub\022\023\n\013destin",
+      "ation\030\001 \001(\t\022\021\n\tclient_id\030\002 \001(\t\022\016\n\006filter" +
+      "\030\007 \001(\t\"?\n\005Unsub\022\023\n\013destination\030\001 \001(\t\022\021\n\t" +
+      "client_id\030\002 \001(\t\022\016\n\006filter\030\007 \001(\t\"\200\001\n\003Get\022" +
+      "\023\n\013destination\030\001 \001(\t\022\021\n\tclient_id\030\002 \001(\t\022" +
+      "\022\n\nfetch_size\030\003 \001(\005\022\023\n\007timeout\030\004 \001(\003:\002-1" +
+      "\022\017\n\004unit\030\005 \001(\005:\0012\022\027\n\010auto_ack\030\006 \001(\010:\005fal" +
+      "se\".\n\010Messages\022\020\n\010batch_id\030\001 \001(\003\022\020\n\010mess" +
+      "ages\030\002 \003(\014\"?\n\004Dump\022\017\n\007journal\030\001 \001(\t\022\020\n\010p" +
+      "osition\030\002 \001(\003\022\024\n\ttimestamp\030\003 \001(\003:\0010\"J\n\016C" +
+      "lientRollback\022\023\n\013destination\030\001 \001(\t\022\021\n\tcl",
+      "ient_id\030\002 \001(\t\022\020\n\010batch_id\030\003 \001(\003*4\n\013Compr" +
+      "ession\022\010\n\004NONE\020\001\022\010\n\004ZLIB\020\002\022\010\n\004GZIP\020\003\022\007\n\003" +
+      "LZF\020\004*\305\001\n\nPacketType\022\r\n\tHANDSHAKE\020\001\022\030\n\024C" +
+      "LIENTAUTHENTICATION\020\002\022\007\n\003ACK\020\003\022\020\n\014SUBSCR" +
+      "IPTION\020\004\022\022\n\016UNSUBSCRIPTION\020\005\022\007\n\003GET\020\006\022\014\n" +
+      "\010MESSAGES\020\007\022\r\n\tCLIENTACK\020\010\022\014\n\010SHUTDOWN\020\t" +
+      "\022\010\n\004DUMP\020\n\022\r\n\tHEARTBEAT\020\013\022\022\n\016CLIENTROLLB" +
+      "ACK\020\014B1\n com.alibaba.otter.canal.protoco" +
+      "lB\013CanalPacketH\001"
+    };
+    com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
+      new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+        public com.google.protobuf.ExtensionRegistry assignDescriptors(
+            com.google.protobuf.Descriptors.FileDescriptor root) {
+          descriptor = root;
+          internal_static_com_alibaba_otter_canal_protocol_Packet_descriptor =
+            getDescriptor().getMessageTypes().get(0);
+          internal_static_com_alibaba_otter_canal_protocol_Packet_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Packet_descriptor,
+              new java.lang.String[] { "MagicNumber", "Version", "Type", "Compression", "Body", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Packet.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Packet.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_HeartBeat_descriptor =
+            getDescriptor().getMessageTypes().get(1);
+          internal_static_com_alibaba_otter_canal_protocol_HeartBeat_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_HeartBeat_descriptor,
+              new java.lang.String[] { "SendTimestamp", "StartTimestamp", },
+              com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.HeartBeat.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Handshake_descriptor =
+            getDescriptor().getMessageTypes().get(2);
+          internal_static_com_alibaba_otter_canal_protocol_Handshake_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Handshake_descriptor,
+              new java.lang.String[] { "CommunicationEncoding", "Seeds", "SupportedCompressions", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Handshake.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Handshake.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_ClientAuth_descriptor =
+            getDescriptor().getMessageTypes().get(3);
+          internal_static_com_alibaba_otter_canal_protocol_ClientAuth_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_ClientAuth_descriptor,
+              new java.lang.String[] { "Username", "Password", "NetReadTimeout", "NetWriteTimeout", "Destination", "ClientId", "Filter", "StartTimestamp", },
+              com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Ack_descriptor =
+            getDescriptor().getMessageTypes().get(4);
+          internal_static_com_alibaba_otter_canal_protocol_Ack_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Ack_descriptor,
+              new java.lang.String[] { "ErrorCode", "ErrorMessage", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Ack.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Ack.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_ClientAck_descriptor =
+            getDescriptor().getMessageTypes().get(5);
+          internal_static_com_alibaba_otter_canal_protocol_ClientAck_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_ClientAck_descriptor,
+              new java.lang.String[] { "Destination", "ClientId", "BatchId", },
+              com.alibaba.otter.canal.protocol.CanalPacket.ClientAck.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.ClientAck.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Sub_descriptor =
+            getDescriptor().getMessageTypes().get(6);
+          internal_static_com_alibaba_otter_canal_protocol_Sub_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Sub_descriptor,
+              new java.lang.String[] { "Destination", "ClientId", "Filter", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Sub.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Sub.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Unsub_descriptor =
+            getDescriptor().getMessageTypes().get(7);
+          internal_static_com_alibaba_otter_canal_protocol_Unsub_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Unsub_descriptor,
+              new java.lang.String[] { "Destination", "ClientId", "Filter", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Unsub.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Unsub.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Get_descriptor =
+            getDescriptor().getMessageTypes().get(8);
+          internal_static_com_alibaba_otter_canal_protocol_Get_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Get_descriptor,
+              new java.lang.String[] { "Destination", "ClientId", "FetchSize", "Timeout", "Unit", "AutoAck", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Get.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Get.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Messages_descriptor =
+            getDescriptor().getMessageTypes().get(9);
+          internal_static_com_alibaba_otter_canal_protocol_Messages_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Messages_descriptor,
+              new java.lang.String[] { "BatchId", "Messages", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Messages.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Messages.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_Dump_descriptor =
+            getDescriptor().getMessageTypes().get(10);
+          internal_static_com_alibaba_otter_canal_protocol_Dump_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_Dump_descriptor,
+              new java.lang.String[] { "Journal", "Position", "Timestamp", },
+              com.alibaba.otter.canal.protocol.CanalPacket.Dump.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.Dump.Builder.class);
+          internal_static_com_alibaba_otter_canal_protocol_ClientRollback_descriptor =
+            getDescriptor().getMessageTypes().get(11);
+          internal_static_com_alibaba_otter_canal_protocol_ClientRollback_fieldAccessorTable = new
+            com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+              internal_static_com_alibaba_otter_canal_protocol_ClientRollback_descriptor,
+              new java.lang.String[] { "Destination", "ClientId", "BatchId", },
+              com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback.class,
+              com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback.Builder.class);
+          return null;
+        }
+      };
+    com.google.protobuf.Descriptors.FileDescriptor
+      .internalBuildGeneratedFileFrom(descriptorData,
+        new com.google.protobuf.Descriptors.FileDescriptor[] {
+        }, assigner);
+  }
+  
+  // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalProtocol.proto b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalProtocol.proto
new file mode 100644
index 00000000..80f03439
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalProtocol.proto
@@ -0,0 +1,114 @@
+package com.alibaba.otter.canal.protocol;
+
+option java_package = "com.alibaba.otter.canal.protocol";
+option java_outer_classname = "CanalPacket";
+option optimize_for = SPEED;
+
+enum Compression {
+    NONE = 1;
+    ZLIB = 2;
+    GZIP = 3;
+    LZF = 4;
+}
+
+enum PacketType {
+    HANDSHAKE = 1;
+    CLIENTAUTHENTICATION = 2;
+    ACK = 3;
+    SUBSCRIPTION = 4;
+    UNSUBSCRIPTION = 5;
+    GET = 6;
+    MESSAGES = 7;
+    CLIENTACK = 8;
+    // management part
+    SHUTDOWN = 9;
+    // integration
+    DUMP = 10;
+    HEARTBEAT = 11;
+    CLIENTROLLBACK = 12;
+}
+
+message Packet {
+    optional int32 magic_number = 1 [default = 17];
+    optional int32 version = 2 [default = 1];
+    optional PacketType type = 3;
+    optional Compression compression = 4 [default = NONE];
+    optional bytes body = 5;
+}
+
+message HeartBeat {
+    optional int64 send_timestamp = 1;
+    optional int64 start_timestamp = 2;
+}
+
+message Handshake {
+    optional string communication_encoding = 1 [default = "utf8"];
+    optional bytes seeds = 2;
+    repeated Compression supported_compressions = 3;
+}
+
+// client authentication
+message ClientAuth {
+    optional string username = 1;
+    optional bytes password = 2; // hashed password with seeds from Handshake message
+    optional int32 net_read_timeout = 3 [default = 0]; // in seconds
+    optional int32 net_write_timeout = 4 [default = 0]; // in seconds
+    optional string destination = 5;
+    optional string client_id = 6;
+    optional string filter = 7;
+    optional int64 start_timestamp = 8;
+}
+
+message Ack {
+    optional int32 error_code = 1 [default = 0];
+    optional string error_message = 2; // if something like compression is not supported, erorr_message will tell about it.
+}
+
+message ClientAck {
+    optional string destination = 1;
+    optional string client_id = 2;
+    optional int64 batch_id = 3;
+}
+
+// subscription
+message Sub {
+    optional string destination = 1;
+    optional string client_id = 2;
+    optional string filter = 7;
+}
+
+// Unsubscription
+message Unsub {
+    optional string destination = 1;
+    optional string client_id = 2;
+    optional string filter = 7;
+}
+
+//  PullRequest
+message Get {
+    optional string destination = 1;
+    optional string client_id = 2;
+    optional int32 fetch_size = 3;
+    optional int64 timeout = 4 [default = -1]; // 默认-1时代表不控制
+    optional int32 unit = 5 [default = 2];// 数字类型，0:纳秒,1:毫秒,2:微秒,3:秒,4:分钟,5:小时,6:天
+    optional bool auto_ack = 6 [default = false]; // 是否自动ack
+}
+
+//
+message Messages {
+	optional int64 batch_id = 1;
+    repeated bytes messages = 2;
+}
+
+// TBD when new packets are required
+message Dump{
+    optional string journal = 1;
+    optional int64  position = 2;
+    optional int64 timestamp = 3 [default = 0];
+}
+
+message ClientRollback{
+    optional string destination = 1;
+    optional string client_id = 2;
+    optional int64 batch_id = 3;
+}
\ No newline at end of file
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java
new file mode 100644
index 00000000..52e3fb05
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/ClientIdentity.java
@@ -0,0 +1,102 @@
+package com.alibaba.otter.canal.protocol;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * @author zebin.xuzb @ 2012-6-20
+ * @version 1.0.0
+ */
+public class ClientIdentity {
+
+    private String destination;
+    private short  clientId;
+    private String filter;
+
+    public ClientIdentity(){
+
+    }
+
+    public ClientIdentity(String destination, short clientId){
+        this.clientId = clientId;
+        this.destination = destination;
+    }
+
+    public ClientIdentity(String destination, short clientId, String filter){
+        this.clientId = clientId;
+        this.destination = destination;
+        this.filter = filter;
+    }
+
+    public Boolean hasFilter() {
+        if (filter == null) {
+            return false;
+        }
+        return StringUtils.isNotBlank(filter);
+    }
+
+    // ======== setter =========
+
+    public String getDestination() {
+        return destination;
+    }
+
+    public short getClientId() {
+        return clientId;
+    }
+
+    public void setClientId(short clientId) {
+        this.clientId = clientId;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public String getFilter() {
+        return filter;
+    }
+
+    public void setFilter(String filter) {
+        this.filter = filter;
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + clientId;
+        result = prime * result + ((destination == null) ? 0 : destination.hashCode());
+        return result;
+    }
+
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof ClientIdentity)) {
+            return false;
+        }
+        ClientIdentity other = (ClientIdentity) obj;
+        if (clientId != other.clientId) {
+            return false;
+        }
+        if (destination == null) {
+            if (other.destination != null) {
+                return false;
+            }
+        } else if (!destination.equals(other.destination)) {
+            return false;
+        }
+        return true;
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/EntryProtocol.proto b/protocol/src/main/java/com/alibaba/otter/canal/protocol/EntryProtocol.proto
new file mode 100644
index 00000000..4553556b
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/EntryProtocol.proto
@@ -0,0 +1,196 @@
+package com.alibaba.otter.canal.protocol;
+
+option java_package = "com.alibaba.otter.canal.protocol";
+option java_outer_classname = "CanalEntry";
+option optimize_for = SPEED;
+
+/****************************************************************
+ * message model
+ *如果要在Enum中新增类型，确保以前的类型的下标值不变.
+ ****************************************************************/
+message Entry {
+	/**协议头部信息**/
+	optional Header						header 				= 1;
+	
+	/**打散后的事件类型**/
+	optional EntryType					entryType			= 2 [default = ROWDATA];
+	
+	/**传输的二进制数组**/
+	optional bytes						storeValue			= 3;
+}
+
+/**message Header**/
+message Header {
+	/**协议的版本号**/
+	optional int32 					version				= 1 [default = 1];	
+	
+	/**binlog/redolog 文件名**/
+	optional string					logfileName			= 2;
+	
+	/**binlog/redolog 文件的偏移位置**/
+	optional int64 					logfileOffset		= 3;
+	
+	/**服务端serverId**/
+	optional 	int64				serverId         	= 4;	
+	
+	/** 变更数据的编码 **/
+	optional string					serverenCode		= 5;
+	
+	/**变更数据的执行时间 **/
+	optional int64					executeTime			= 6;
+	
+	/** 变更数据的来源**/
+	optional Type					sourceType			= 7 [default = MYSQL];
+	
+	/** 变更数据的schemaname**/
+	optional string					schemaName			= 8;
+	
+	/**变更数据的tablename**/
+	optional string					tableName			= 9;	
+	
+	/**每个event的长度**/
+	optional int64					eventLength         = 10;
+	
+	/**数据变更类型**/
+	optional EventType 				eventType			= 11 [default = UPDATE];
+	
+	/**预留扩展**/
+	repeated Pair					props				= 12;		
+}
+
+/**每个字段的数据结构**/
+message Column {
+	/**字段下标**/
+	optional int32		index			= 		1;
+	
+	/**字段java中类型**/
+	optional int32 		sqlType			= 		2;
+	
+	/**字段名称(忽略大小写)，在mysql中是没有的**/
+	optional string		name			=		3;
+	
+	/**是否是主键**/
+	optional bool 		isKey			= 		4;
+	
+	/**如果EventType=UPDATE,用于标识这个字段值是否有修改**/
+	optional bool		updated			= 		5;
+	
+	/** 标识是否为空  **/
+	optional bool		isNull			= 		6 [default = false];
+	
+	/**预留扩展**/
+	repeated Pair		props			=		7;	
+	
+	/** 字段值,timestamp,Datetime是一个时间格式的文本 **/
+	optional string		value			= 		8;
+	
+	/** 对应数据对象原始长度 **/
+	optional int32		length			= 		9;
+	
+	/**字段mysql类型**/
+	optional string		mysqlType		= 		10;
+}
+
+message RowData {
+
+	/** 字段信息，增量数据(修改前,删除前) **/
+	repeated Column			beforeColumns  	= 		1;
+
+	/** 字段信息，增量数据(修改后,新增后)  **/
+	repeated Column			afterColumns	= 		2;
+	
+	/**预留扩展**/
+	repeated Pair			props			=		3;	
+}
+
+/**message row 每行变更数据的数据结构**/
+message RowChange {
+
+	/**tableId,由数据库产生**/
+	optional int64	 		tableId			=		1;
+	
+	/**数据变更类型**/
+	optional EventType 		eventType		= 		2 [default = UPDATE];
+	
+	/** 标识是否是ddl语句  **/
+	optional bool			isDdl			= 		10 [default = false];
+
+	/** ddl/query的sql语句  **/
+	optional string			sql 			= 		11;
+	
+	/** 一次数据库变更可能存在多行  **/
+	repeated RowData		rowDatas		= 		12;
+	
+	/**预留扩展**/
+	repeated Pair			props			=		13;	
+	
+	/** ddl/query的schemaName，会存在跨库ddl，需要保留执行ddl的当前schemaName  **/
+	optional string			ddlSchemaName 	= 		14;
+}
+
+/**开始事务的一些信息**/
+message TransactionBegin{
+	
+	/**已废弃，请使用header里的executeTime**/
+	optional int64			executeTime		=		1;
+	
+	/**已废弃，Begin里不提供事务id**/
+	optional string			transactionId	=		2;
+	
+	/**预留扩展**/
+	repeated Pair			props			=		3;	
+	
+	/**执行的thread Id**/
+	optional int64			threadId		=		4;
+}
+
+/**结束事务的一些信息**/
+message TransactionEnd{
+	
+	/**已废弃，请使用header里的executeTime**/
+	optional int64			executeTime		=		1;
+	
+	/**事务号**/
+	optional string			transactionId	=		2;
+	
+	/**预留扩展**/
+	repeated Pair			props			=		3;	
+}
+
+/**预留扩展**/
+message Pair{
+	optional string 		key				= 			1;	
+	optional string 		value			= 			2;	
+}
+
+/**打散后的事件类型，主要用于标识事务的开始，变更数据，结束**/
+enum EntryType{
+	TRANSACTIONBEGIN 		=		1;
+	ROWDATA					=		2;
+	TRANSACTIONEND			=		3;
+	/** 心跳类型，内部使用，外部暂不可见，可忽略 **/
+	HEARTBEAT				=		4; 
+}
+
+/** 事件类型 **/
+enum EventType {
+    INSERT 		= 		1;
+    UPDATE 		= 		2;
+    DELETE 		= 		3;
+    CREATE		= 		4;
+    ALTER		= 		5;
+    ERASE		= 		6;
+    QUERY		=		7;
+    TRUNCATE	=		8;
+    RENAME 		= 		9;
+    /**CREATE INDEX**/
+    CINDEX		= 		10;
+    DINDEX 		= 		11;
+}
+
+/**数据库类型**/
+enum Type {
+    ORACLE		= 		1;
+    MYSQL		= 		2;
+    PGSQL		= 		3;
+}
\ No newline at end of file
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java
new file mode 100644
index 00000000..984766e2
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/Message.java
@@ -0,0 +1,56 @@
+package com.alibaba.otter.canal.protocol;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+
+/**
+ * @author zebin.xuzb @ 2012-6-19
+ * @version 1.0.0
+ */
+public class Message implements Serializable {
+
+    private static final long      serialVersionUID = 1234034768477580009L;
+
+    private long                   id;
+    private List entries          = new ArrayList();
+
+    public Message(long id, List entries){
+        this.id = id;
+        this.entries = entries == null ? new ArrayList() : entries;
+    }
+
+    public Message(long id){
+        this.id = id;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    public List getEntries() {
+        return entries;
+    }
+
+    public void setEntries(List entries) {
+        this.entries = entries;
+    }
+
+    public void addEntry(CanalEntry.Entry entry) {
+        this.entries.add(entry);
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/exception/CanalClientException.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/exception/CanalClientException.java
new file mode 100644
index 00000000..72b77ef7
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/exception/CanalClientException.java
@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.protocol.exception;
+
+import org.apache.commons.lang.exception.NestableRuntimeException;
+
+/**
+ * @author zebin.xuzb @ 2012-6-20
+ * @version 1.0.0
+ */
+public class CanalClientException extends NestableRuntimeException {
+
+    private static final long serialVersionUID = -7545341502620139031L;
+
+    public CanalClientException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalClientException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalClientException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalClientException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalClientException(Throwable cause){
+        super(cause);
+    }
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java
new file mode 100644
index 00000000..33d31a05
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/EntryPosition.java
@@ -0,0 +1,109 @@
+package com.alibaba.otter.canal.protocol.position;
+
+/**
+ * 数据库对象的唯一标示
+ * 
+ * @author jianghang 2012-6-14 下午09:20:07
+ * @version 1.0.0
+ */
+public class EntryPosition extends TimePosition {
+
+    private static final long serialVersionUID      = 81432665066427482L;
+    public static final int   EVENTIDENTITY_SEGMENT = 3;
+    public static final char  EVENTIDENTITY_SPLIT   = (char) 5;
+
+    private boolean           included              = false;
+    private String            journalName;
+    private Long              position;
+
+    public EntryPosition(){
+        super(null);
+    }
+
+    public EntryPosition(Long timestamp){
+        this(null, null, timestamp);
+    }
+
+    public EntryPosition(String journalName, Long position){
+        this(journalName, position, null);
+    }
+
+    public EntryPosition(String journalName, Long position, Long timestamp){
+        super(timestamp);
+        this.journalName = journalName;
+        this.position = position;
+    }
+
+    public String getJournalName() {
+        return journalName;
+    }
+
+    public void setJournalName(String journalName) {
+        this.journalName = journalName;
+    }
+
+    public Long getPosition() {
+        return position;
+    }
+
+    public void setPosition(Long position) {
+        this.position = position;
+    }
+
+    public boolean isIncluded() {
+        return included;
+    }
+
+    public void setIncluded(boolean included) {
+        this.included = included;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = super.hashCode();
+        result = prime * result + ((journalName == null) ? 0 : journalName.hashCode());
+        result = prime * result + ((position == null) ? 0 : position.hashCode());
+        // 手写equals，自动生成时需注意
+        result = prime * result + ((timestamp == null) ? 0 : timestamp.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!super.equals(obj)) {
+            return false;
+        }
+        if (!(obj instanceof EntryPosition)) {
+            return false;
+        }
+        EntryPosition other = (EntryPosition) obj;
+        if (journalName == null) {
+            if (other.journalName != null) {
+                return false;
+            }
+        } else if (!journalName.equals(other.journalName)) {
+            return false;
+        }
+        if (position == null) {
+            if (other.position != null) {
+                return false;
+            }
+        } else if (!position.equals(other.position)) {
+            return false;
+        }
+        // 手写equals，自动生成时需注意
+        if (timestamp == null) {
+            if (other.timestamp != null) {
+                return false;
+            }
+        } else if (!timestamp.equals(other.timestamp)) {
+            return false;
+        }
+        return true;
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogIdentity.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogIdentity.java
new file mode 100644
index 00000000..56410a9d
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogIdentity.java
@@ -0,0 +1,72 @@
+package com.alibaba.otter.canal.protocol.position;
+
+import java.net.InetSocketAddress;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * @author jianghang 2012-6-21 上午10:52:02
+ * @version 1.0.0
+ */
+public class LogIdentity extends Position {
+
+    private static final long serialVersionUID = 5530225131455662581L;
+    private InetSocketAddress sourceAddress;                          // 链接服务器的地址
+    private Long              slaveId;                                // 对应的slaveId
+
+    public LogIdentity(){
+    }
+
+    public LogIdentity(InetSocketAddress sourceAddress, Long slaveId){
+        this.sourceAddress = sourceAddress;
+        this.slaveId = slaveId;
+    }
+
+    public InetSocketAddress getSourceAddress() {
+        return sourceAddress;
+    }
+
+    public void setSourceAddress(InetSocketAddress sourceAddress) {
+        this.sourceAddress = sourceAddress;
+    }
+
+    public Long getSlaveId() {
+        return slaveId;
+    }
+
+    public void setSlaveId(Long slaveId) {
+        this.slaveId = slaveId;
+    }
+
+    @Override
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((slaveId == null) ? 0 : slaveId.hashCode());
+        result = prime * result + ((sourceAddress == null) ? 0 : sourceAddress.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        LogIdentity other = (LogIdentity) obj;
+        if (slaveId == null) {
+            if (other.slaveId != null) return false;
+        } else if (slaveId.longValue() != (other.slaveId.longValue())) return false;
+        if (sourceAddress == null) {
+            if (other.sourceAddress != null) return false;
+        } else if (!sourceAddress.equals(other.sourceAddress)) return false;
+        return true;
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogPosition.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogPosition.java
new file mode 100644
index 00000000..a0602cf1
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/LogPosition.java
@@ -0,0 +1,69 @@
+package com.alibaba.otter.canal.protocol.position;
+
+/**
+ * 基于mysql/oracle log位置标示
+ * 
+ * @author jianghang 2012-6-21 上午10:52:41
+ * @version 1.0.0
+ */
+public class LogPosition extends Position {
+
+    private static final long serialVersionUID = 3875012010277005819L;
+    private LogIdentity       identity;
+    private EntryPosition     postion;
+
+    public LogIdentity getIdentity() {
+        return identity;
+    }
+
+    public void setIdentity(LogIdentity identity) {
+        this.identity = identity;
+    }
+
+    public EntryPosition getPostion() {
+        return postion;
+    }
+
+    public void setPostion(EntryPosition postion) {
+        this.postion = postion;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((identity == null) ? 0 : identity.hashCode());
+        result = prime * result + ((postion == null) ? 0 : postion.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof LogPosition)) {
+            return false;
+        }
+        LogPosition other = (LogPosition) obj;
+        if (identity == null) {
+            if (other.identity != null) {
+                return false;
+            }
+        } else if (!identity.equals(other.identity)) {
+            return false;
+        }
+        if (postion == null) {
+            if (other.postion != null) {
+                return false;
+            }
+        } else if (!postion.equals(other.postion)) {
+            return false;
+        }
+        return true;
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/MetaqPosition.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/MetaqPosition.java
new file mode 100644
index 00000000..a47a552a
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/MetaqPosition.java
@@ -0,0 +1,46 @@
+package com.alibaba.otter.canal.protocol.position;
+
+/**
+ * @author zebin.xuzb 2012-11-3 上午12:23:01
+ * @since 1.0.0
+ */
+public class MetaqPosition extends Position {
+
+    private static final long serialVersionUID = -8673508769040569273L;
+
+    private String            topic;
+    private String            msgNewId;
+    private long              offset;
+
+    public MetaqPosition(String topic, String msgNewId, long offset){
+        super();
+        this.topic = topic;
+        this.msgNewId = msgNewId;
+        this.offset = offset;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public String getMsgNewId() {
+        return msgNewId;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public void setMsgNewId(String msgNewId) {
+        this.msgNewId = msgNewId;
+    }
+
+    public long getOffset() {
+        return offset;
+    }
+
+    public void setOffset(long offset) {
+        this.offset = offset;
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/Position.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/Position.java
new file mode 100644
index 00000000..92cb8d3c
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/Position.java
@@ -0,0 +1,20 @@
+package com.alibaba.otter.canal.protocol.position;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * 事件唯一标示
+ */
+public abstract class Position implements Serializable {
+
+    private static final long serialVersionUID = 2332798099928474975L;
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/PositionRange.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/PositionRange.java
new file mode 100644
index 00000000..abfac4b2
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/PositionRange.java
@@ -0,0 +1,106 @@
+package com.alibaba.otter.canal.protocol.position;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * 描述一个position范围
+ * 
+ * @author jianghang 2012-7-10 下午05:28:38
+ * @version 1.0.0
+ */
+public class PositionRange implements Serializable {
+
+    private static final long serialVersionUID = -9162037079815694784L;
+    private T                 start;
+    // add by ljh at 2012-09-05，用于记录一个可被ack的位置，保证每次提交到cursor中的位置是一个完整事务的结束
+    private T                 ack;
+    private T                 end;
+
+    public PositionRange(){
+    }
+
+    public PositionRange(T start, T end){
+        this.start = start;
+        this.end = end;
+    }
+
+    public T getStart() {
+        return start;
+    }
+
+    public void setStart(T start) {
+        this.start = start;
+    }
+
+    public T getEnd() {
+        return end;
+    }
+
+    public void setEnd(T end) {
+        this.end = end;
+    }
+
+    public T getAck() {
+        return ack;
+    }
+
+    public void setAck(T ack) {
+        this.ack = ack;
+    }
+
+    @Override
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((ack == null) ? 0 : ack.hashCode());
+        result = prime * result + ((end == null) ? 0 : end.hashCode());
+        result = prime * result + ((start == null) ? 0 : start.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof PositionRange)) {
+            return false;
+        }
+        PositionRange other = (PositionRange) obj;
+        if (ack == null) {
+            if (other.ack != null) {
+                return false;
+            }
+        } else if (!ack.equals(other.ack)) {
+            return false;
+        }
+        if (end == null) {
+            if (other.end != null) {
+                return false;
+            }
+        } else if (!end.equals(other.end)) {
+            return false;
+        }
+        if (start == null) {
+            if (other.start != null) {
+                return false;
+            }
+        } else if (!start.equals(other.start)) {
+            return false;
+        }
+        return true;
+    }
+
+}
diff --git a/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/TimePosition.java b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/TimePosition.java
new file mode 100644
index 00000000..e48585d6
--- /dev/null
+++ b/protocol/src/main/java/com/alibaba/otter/canal/protocol/position/TimePosition.java
@@ -0,0 +1,56 @@
+package com.alibaba.otter.canal.protocol.position;
+
+/**
+ * 基于时间的位置，position数据不唯一
+ * 
+ * @author jianghang 2012-6-14 下午09:22:04
+ * @version 1.0.0
+ */
+public class TimePosition extends Position {
+
+    private static final long serialVersionUID = 6185261261064226380L;
+    protected Long            timestamp;
+
+    public TimePosition(Long timestamp){
+        this.timestamp = timestamp;
+    }
+
+    public Long getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(Long timestamp) {
+        this.timestamp = timestamp;
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + ((timestamp == null) ? 0 : timestamp.hashCode());
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (!(obj instanceof TimePosition)) {
+            return false;
+        }
+        TimePosition other = (TimePosition) obj;
+        if (timestamp == null) {
+            if (other.timestamp != null) {
+                return false;
+            }
+        } else if (!timestamp.equals(other.timestamp)) {
+            return false;
+        }
+        return true;
+    }
+
+}
diff --git a/server/pom.xml b/server/pom.xml
new file mode 100644
index 00000000..b4b0da4f
--- /dev/null
+++ b/server/pom.xml
@@ -0,0 +1,37 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.server
+	jar
+	canal server module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.instance.core
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.instance.spring
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.instance.manager
+			${project.version}
+		
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/CanalServer.java b/server/src/main/java/com/alibaba/otter/canal/server/CanalServer.java
new file mode 100644
index 00000000..a6453d7b
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/CanalServer.java
@@ -0,0 +1,17 @@
+package com.alibaba.otter.canal.server;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.server.exception.CanalServerException;
+
+/**
+ * 对应canal整个服务实例，一个jvm实例只有一份server
+ * 
+ * @author jianghang 2012-7-12 下午01:32:29
+ * @version 1.0.0
+ */
+public interface CanalServer extends CanalLifeCycle {
+
+    public void start() throws CanalServerException;
+
+    public void stop() throws CanalServerException;
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/embeded/CanalServerWithEmbeded.java b/server/src/main/java/com/alibaba/otter/canal/server/embeded/CanalServerWithEmbeded.java
new file mode 100644
index 00000000..82518928
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/embeded/CanalServerWithEmbeded.java
@@ -0,0 +1,459 @@
+package com.alibaba.otter.canal.server.embeded;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.alibaba.otter.canal.server.CanalServer;
+import com.alibaba.otter.canal.server.exception.CanalServerException;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+
+/**
+ * 嵌入式版本实现
+ * 
+ * @author jianghang 2012-7-12 下午01:34:00
+ * @author zebin.xuzb
+ * @version 1.0.0
+ */
+public class CanalServerWithEmbeded extends AbstractCanalLifeCycle implements CanalServer {
+
+    private static final Logger        logger = LoggerFactory.getLogger(CanalServerWithEmbeded.class);
+    private Map canalInstances;
+    // private Map lastRollbackPostions;
+    private CanalInstanceGenerator     canalInstanceGenerator;
+
+    public void start() {
+        super.start();
+
+        canalInstances = new MapMaker().makeComputingMap(new Function() {
+
+            public CanalInstance apply(String destination) {
+                CanalInstance canalInstance = canalInstanceGenerator.generate(destination);
+                return canalInstance;
+            }
+        });
+
+        // lastRollbackPostions = new MapMaker().makeMap();
+    }
+
+    public void stop() {
+        super.stop();
+        for (Map.Entry entry : canalInstances.entrySet()) {
+            try {
+                CanalInstance instance = entry.getValue();
+                if (instance.isStart()) {
+                    try {
+                        String destination = entry.getKey();
+                        MDC.put("destination", destination);
+                        entry.getValue().stop();
+                        logger.info("stop CanalInstances[{}] successfully", destination);
+                    } finally {
+                        MDC.remove("destination");
+                    }
+                }
+            } catch (Exception e) {
+                logger.error(String.format("stop cannalInstance[%s] has an error", entry.getKey()), e);
+            }
+        }
+    }
+
+    public void start(final String destination) {
+        final CanalInstance canalInstance = canalInstances.get(destination);
+        if (!canalInstance.isStart()) {
+            try {
+                MDC.put("destination", destination);
+                canalInstance.start();
+                logger.info("start CanalInstances[{}] successfully", destination);
+            } finally {
+                MDC.remove("destination");
+            }
+        }
+    }
+
+    public void stop(String destination) {
+        CanalInstance canalInstance = canalInstances.remove(destination);
+        if (canalInstance != null) {
+            if (canalInstance.isStart()) {
+                try {
+                    MDC.put("destination", destination);
+                    canalInstance.stop();
+                    logger.info("stop CanalInstances[{}] successfully", destination);
+                } finally {
+                    MDC.remove("destination");
+                }
+            }
+        }
+    }
+
+    public boolean isStart(String destination) {
+        return canalInstances.containsKey(destination) && canalInstances.get(destination).isStart();
+    }
+
+    /**
+     * 客户端订阅，重复订阅时会更新对应的filter信息
+     */
+    public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        if (!canalInstance.getMetaManager().isStart()) {
+            canalInstance.getMetaManager().start();
+        }
+
+        canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅
+
+        Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
+        if (position == null) {
+            position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
+            if (position != null) {
+                canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
+            }
+            logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
+        } else {
+            logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
+        }
+
+        // 通知下订阅关系变化
+        canalInstance.subscribeChange(clientIdentity);
+    }
+
+    /**
+     * 取消订阅
+     */
+    public void unsubscribe(ClientIdentity clientIdentity) throws CanalServerException {
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        canalInstance.getMetaManager().unsubscribe(clientIdentity); // 执行一下meta订阅
+
+        logger.info("unsubscribe successfully, {}", clientIdentity);
+    }
+
+    /**
+     * 查询所有的订阅信息
+     */
+    public List listAllSubscribe(String destination) throws CanalServerException {
+        CanalInstance canalInstance = canalInstances.get(destination);
+        return canalInstance.getMetaManager().listAllSubscribeInfo(destination);
+    }
+
+    /**
+     * 获取数据
+     * 
+     * 
+     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
+     * 

+     */
+    public Message get(ClientIdentity clientIdentity, int batchSize) throws CanalServerException {
+        return get(clientIdentity, batchSize, null, null);
+    }
+
+    /**
+     * 获取数据，可以指定超时时间.
+     * 
+     * 
+     * 几种case:
+     * a. 如果timeout为null，则采用tryGet方式，即时获取
+     * b. 如果timeout不为null
+     *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
+     *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
+     * 
+     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
+     * 

+     */
+    public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
+                                                                                                 throws CanalServerException {
+        checkStart(clientIdentity.getDestination());
+        checkSubscribe(clientIdentity);
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        synchronized (canalInstance) {
+            // 获取到流式数据中的最后一批获取的位置
+            PositionRange positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
+
+            if (positionRanges != null) {
+                throw new CanalServerException(
+                                               String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data",
+                                                             clientIdentity.getClientId(), positionRanges));
+            }
+
+            Events events = null;
+            Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
+            events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
+
+            if (CollectionUtils.isEmpty(events.getEvents())) {
+                logger.debug("get successfully, clientId:{} batchSize:{} but result is null", new Object[] {
+                        clientIdentity.getClientId(), batchSize });
+                return new Message(-1, new ArrayList()); // 返回空包，避免生成batchId，浪费性能
+            } else {
+                // 记录到流式信息
+                Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
+                List entrys = Lists.transform(events.getEvents(), new Function() {
+
+                    public Entry apply(Event input) {
+                        return input.getEntry();
+                    }
+                });
+
+                logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
+                            new Object[] { clientIdentity.getClientId(), batchSize, entrys.size(), batchId,
+                                    events.getPositionRange() });
+                // 直接提交ack
+                ack(clientIdentity, batchId);
+                return new Message(batchId, entrys);
+            }
+        }
+    }
+
+    /**
+     * 不指定 position 获取事件。canal 会记住此 client 最新的 position。 

+     * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
+     * 
+     * 
+     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
+     * 

+     */
+    public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize) throws CanalServerException {
+        return getWithoutAck(clientIdentity, batchSize, null, null);
+    }
+
+    /**
+     * 不指定 position 获取事件。canal 会记住此 client 最新的 position。 

+     * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
+     * 
+     * 
+     * 几种case:
+     * a. 如果timeout为null，则采用tryGet方式，即时获取
+     * b. 如果timeout不为null
+     *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
+     *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
+     *    
+     * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
+     * 

+     */
+    public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
+                                                                                                           throws CanalServerException {
+        checkStart(clientIdentity.getDestination());
+        checkSubscribe(clientIdentity);
+
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        synchronized (canalInstance) {
+            // 获取到流式数据中的最后一批获取的位置
+            PositionRange positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
+
+            Events events = null;
+            if (positionRanges != null) { // 存在流数据
+                events = getEvents(canalInstance.getEventStore(), positionRanges.getStart(), batchSize, timeout, unit);
+            } else {// ack后第一次获取
+                Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
+                if (start == null) { // 第一次，还没有过ack记录，则获取当前store中的第一条
+                    start = canalInstance.getEventStore().getFirstPosition();
+                }
+
+                events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
+            }
+
+            if (CollectionUtils.isEmpty(events.getEvents())) {
+                logger.debug("getWithoutAck successfully, clientId:{} batchSize:{} but result is null", new Object[] {
+                        clientIdentity.getClientId(), batchSize });
+                return new Message(-1, new ArrayList()); // 返回空包，避免生成batchId，浪费性能
+            } else {
+                // 记录到流式信息
+                Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
+                List entrys = Lists.transform(events.getEvents(), new Function() {
+
+                    public Entry apply(Event input) {
+                        return input.getEntry();
+                    }
+                });
+
+                logger.info("getWithoutAck successfully, clientId:{} batchSize:{}  real size is {} and result is [batchId:{} , position:{}]",
+                            new Object[] { clientIdentity.getClientId(), batchSize, entrys.size(), batchId,
+                                    events.getPositionRange() });
+                return new Message(batchId, entrys);
+            }
+
+        }
+    }
+
+    /**
+     * 查询当前未被ack的batch列表，batchId会按照从小到大进行返回
+     */
+    public List listBatchIds(ClientIdentity clientIdentity) throws CanalServerException {
+        checkStart(clientIdentity.getDestination());
+        checkSubscribe(clientIdentity);
+
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        Map batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity);
+        List result = new ArrayList(batchs.keySet());
+        Collections.sort(result);
+        return result;
+    }
+
+    /**
+     * 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
+     * 
+     * 
+     * 注意：进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
+     * 

+     */
+    public void ack(ClientIdentity clientIdentity, long batchId) throws CanalServerException {
+        checkStart(clientIdentity.getDestination());
+        checkSubscribe(clientIdentity);
+
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        PositionRange positionRanges = null;
+        positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId); // 更新位置
+        if (positionRanges == null) { // 说明是重复的ack/rollback
+            throw new CanalServerException(
+                                           String.format("ack error , clientId:%s batchId:%d is not exist , please check",
+                                                         clientIdentity.getClientId(), batchId));
+        }
+
+        // 更新cursor最好严格判断下位置是否有跳跃更新
+        // Position position = lastRollbackPostions.get(clientIdentity);
+        // if (position != null) {
+        // // Position position =
+        // canalInstance.getMetaManager().getCursor(clientIdentity);
+        // LogPosition minPosition =
+        // CanalEventUtils.min(positionRanges.getStart(), (LogPosition)
+        // position);
+        // if (minPosition == position) {// ack的position要晚于该最后ack的位置，可能有丢数据
+        // throw new CanalServerException(
+        // String.format(
+        // "ack error , clientId:%s batchId:%d %s is jump ack , last ack:%s",
+        // clientIdentity.getClientId(), batchId, positionRanges,
+        // position));
+        // }
+        // }
+
+        // 更新cursor
+        if (positionRanges.getAck() != null) {
+            canalInstance.getMetaManager().updateCursor(clientIdentity, positionRanges.getAck());
+            logger.info("ack successfully, clientId:{} batchId:{} position:{}",
+                        new Object[] { clientIdentity.getClientId(), batchId, positionRanges });
+        }
+
+        // 可定时清理数据
+        canalInstance.getEventStore().ack(positionRanges.getEnd());
+
+    }
+
+    /**
+     * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
+     */
+    public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
+        checkStart(clientIdentity.getDestination());
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
+        boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
+        if (!hasSubscribe) {
+            return;
+        }
+
+        synchronized (canalInstance) {
+            // 清除batch信息
+            canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
+            // rollback eventStore中的状态信息
+            canalInstance.getEventStore().rollback();
+            logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
+        }
+    }
+
+    /**
+     * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
+     */
+    public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
+        checkStart(clientIdentity.getDestination());
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+
+        // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
+        boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
+        if (!hasSubscribe) {
+            return;
+        }
+        synchronized (canalInstance) {
+            // 清除batch信息
+            PositionRange positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
+                                                                                                   batchId);
+            if (positionRanges == null) { // 说明是重复的ack/rollback
+                throw new CanalServerException(
+                                               String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
+                                                             clientIdentity.getClientId(), batchId));
+            }
+
+            // lastRollbackPostions.put(clientIdentity,
+            // positionRanges.getEnd());// 记录一下最后rollback的位置
+            // TODO 后续rollback到指定的batchId位置
+            canalInstance.getEventStore().rollback();// rollback
+                                                     // eventStore中的状态信息
+            logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
+                        new Object[] { clientIdentity.getClientId(), batchId, positionRanges });
+        }
+    }
+
+    public Map getCanalInstances() {
+        return Maps.newHashMap(canalInstances);
+    }
+
+    // ======================== helper method =======================
+
+    /**
+     * 根据不同的参数，选择不同的方式获取数据
+     */
+    private Events getEvents(CanalEventStore eventStore, Position start, int batchSize, Long timeout,
+                                    TimeUnit unit) {
+        if (timeout == null) {
+            return eventStore.tryGet(start, batchSize);
+        } else {
+            try {
+                if (timeout <= 0) {
+                    return eventStore.get(start, batchSize);
+                } else {
+                    return eventStore.get(start, batchSize, timeout, unit);
+                }
+            } catch (Exception e) {
+                throw new CanalServerException(e);
+            }
+        }
+    }
+
+    private void checkSubscribe(ClientIdentity clientIdentity) {
+        CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
+        boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
+        if (!hasSubscribe) {
+            throw new CanalServerException(String.format("ClientIdentity:%s should subscribe first",
+                                                         clientIdentity.toString()));
+        }
+    }
+
+    private void checkStart(String destination) {
+        if (!isStart(destination)) {
+            throw new CanalServerException(String.format("destination:%s should start first", destination));
+        }
+    }
+
+    // ========= setter ==========
+
+    public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) {
+        this.canalInstanceGenerator = canalInstanceGenerator;
+    }
+
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/exception/CanalServerException.java b/server/src/main/java/com/alibaba/otter/canal/server/exception/CanalServerException.java
new file mode 100644
index 00000000..5e4fa0aa
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/exception/CanalServerException.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.server.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalServerException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalServerException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalServerException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalServerException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalServerException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalServerException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNetty.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNetty.java
new file mode 100644
index 00000000..0e39bd48
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/CanalServerWithNetty.java
@@ -0,0 +1,105 @@
+package com.alibaba.otter.canal.server.netty;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.lang.StringUtils;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.server.CanalServer;
+import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.netty.handler.ClientAuthenticationHandler;
+import com.alibaba.otter.canal.server.netty.handler.FixedHeaderFrameDecoder;
+import com.alibaba.otter.canal.server.netty.handler.HandshakeInitializationHandler;
+import com.alibaba.otter.canal.server.netty.handler.SessionHandler;
+
+/**
+ * 基于netty网络服务的server实现
+ * 
+ * @author jianghang 2012-7-12 下午01:34:49
+ * @version 1.0.0
+ */
+public class CanalServerWithNetty extends AbstractCanalLifeCycle implements CanalServer {
+
+    private CanalServerWithEmbeded embededServer;       // 嵌入式server
+    private String                 ip;
+    private int                    port;
+    private Channel                serverChannel = null;
+    private ServerBootstrap        bootstrap     = null;
+
+    public CanalServerWithNetty(){
+    }
+
+    public CanalServerWithNetty(CanalServerWithEmbeded embededServer){
+        this.embededServer = embededServer;
+    }
+
+    public void start() {
+        super.start();
+
+        if (!embededServer.isStart()) {
+            embededServer.start();
+        }
+
+        this.bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),
+                                                                               Executors.newCachedThreadPool()));
+
+        // 构造对应的pipeline
+        bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
+
+            public ChannelPipeline getPipeline() throws Exception {
+                ChannelPipeline pipelines = Channels.pipeline();
+                pipelines.addLast(FixedHeaderFrameDecoder.class.getName(), new FixedHeaderFrameDecoder());
+                pipelines.addLast(HandshakeInitializationHandler.class.getName(), new HandshakeInitializationHandler());
+                pipelines.addLast(ClientAuthenticationHandler.class.getName(),
+                                  new ClientAuthenticationHandler(embededServer));
+
+                SessionHandler sessionHandler = new SessionHandler(embededServer);
+                pipelines.addLast(SessionHandler.class.getName(), sessionHandler);
+                return pipelines;
+            }
+        });
+
+        // 启动
+        if (StringUtils.isNotEmpty(ip)) {
+            this.serverChannel = bootstrap.bind(new InetSocketAddress(this.ip, this.port));
+        } else {
+            this.serverChannel = bootstrap.bind(new InetSocketAddress(this.port));
+        }
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (this.serverChannel != null) {
+            this.serverChannel.close().awaitUninterruptibly(1000);
+        }
+
+        if (this.bootstrap != null) {
+            this.bootstrap.releaseExternalResources();
+        }
+
+        if (embededServer.isStart()) {
+            embededServer.stop();
+        }
+    }
+
+    public void setIp(String ip) {
+        this.ip = ip;
+    }
+
+    public void setPort(int port) {
+        this.port = port;
+    }
+
+    public void setEmbededServer(CanalServerWithEmbeded embededServer) {
+        this.embededServer = embededServer;
+    }
+
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
new file mode 100644
index 00000000..c318d03f
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
@@ -0,0 +1,55 @@
+package com.alibaba.otter.canal.server.netty;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+
+public class NettyUtils {
+
+    private static final Logger logger           = LoggerFactory.getLogger(NettyUtils.class);
+    private static int          HEADER_LENGTH    = 4;
+    public static Timer         hashedWheelTimer = new HashedWheelTimer();
+
+    public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) {
+        byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
+        if (channelFutureListner == null) {
+            Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body));
+        } else {
+            Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body)).addListener(channelFutureListner);
+        }
+    }
+
+    public static void ack(Channel channel, ChannelFutureListener channelFutureListner) {
+        write(
+              channel,
+              Packet.newBuilder().setType(CanalPacket.PacketType.ACK).setBody(Ack.newBuilder().build().toByteString()).build().toByteArray(),
+              channelFutureListner);
+    }
+
+    public static void error(int errorCode, String errorMessage, Channel channel,
+                             ChannelFutureListener channelFutureListener) {
+        if (channelFutureListener == null) {
+            channelFutureListener = ChannelFutureListener.CLOSE;
+        }
+
+        logger.error("ErrotCode:{} , Caused by : \n{}", errorCode, errorMessage);
+        write(
+              channel,
+              Packet.newBuilder().setType(CanalPacket.PacketType.ACK).setBody(
+                                                                              Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(
+                                                                                                                                       errorMessage).build().toByteString()).build().toByteArray(),
+              channelFutureListener);
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java
new file mode 100644
index 00000000..72b4fee2
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java
@@ -0,0 +1,121 @@
+package com.alibaba.otter.canal.server.netty.handler;
+
+import org.apache.commons.lang.StringUtils;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
+import org.jboss.netty.handler.timeout.IdleStateEvent;
+import org.jboss.netty.handler.timeout.IdleStateHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.netty.NettyUtils;
+
+/**
+ * 客户端身份认证处理
+ * 
+ * @author jianghang 2012-10-24 上午11:12:45
+ * @version 1.0.0
+ */
+public class ClientAuthenticationHandler extends SimpleChannelHandler {
+
+    private static final Logger    logger                                  = LoggerFactory.getLogger(ClientAuthenticationHandler.class);
+    private final int              SUPPORTED_VERSION                       = 3;
+    private final int              defaultSubscriptorDisconnectIdleTimeout = 5 * 60 * 1000;
+    private CanalServerWithEmbeded embededServer;
+
+    public ClientAuthenticationHandler(){
+
+    }
+
+    public ClientAuthenticationHandler(CanalServerWithEmbeded embededServer){
+        this.embededServer = embededServer;
+    }
+
+    public void messageReceived(final ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+        ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
+        final Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
+        switch (packet.getVersion()) {
+            case SUPPORTED_VERSION:
+            default:
+                final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
+                // 如果存在订阅信息
+                if (StringUtils.isNotEmpty(clientAuth.getDestination())
+                    && StringUtils.isNotEmpty(clientAuth.getClientId())) {
+                    ClientIdentity clientIdentity = new ClientIdentity(clientAuth.getDestination(),
+                        Short.valueOf(clientAuth.getClientId()),
+                        clientAuth.getFilter());
+                    try {
+                        MDC.put("destination", clientIdentity.getDestination());
+                        embededServer.subscribe(clientIdentity);
+                        ctx.setAttachment(clientIdentity);// 设置状态数据
+                        // 尝试启动，如果已经启动，忽略
+                        if (!embededServer.isStart(clientIdentity.getDestination())) {
+                            ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
+                            if (!runningMonitor.isStart()) {
+                                runningMonitor.start();
+                            }
+                        }
+                    } finally {
+                        MDC.remove("destination");
+                    }
+                }
+
+                NettyUtils.ack(ctx.getChannel(), new ChannelFutureListener() {
+
+                    public void operationComplete(ChannelFuture future) throws Exception {
+                        logger.info("remove unused channel handlers after authentication is done successfully.");
+                        ctx.getPipeline().remove(HandshakeInitializationHandler.class.getName());
+                        ctx.getPipeline().remove(ClientAuthenticationHandler.class.getName());
+
+                        int readTimeout = defaultSubscriptorDisconnectIdleTimeout;
+                        int writeTimeout = defaultSubscriptorDisconnectIdleTimeout;
+                        if (clientAuth.getNetReadTimeout() > 0) {
+                            readTimeout = clientAuth.getNetReadTimeout();
+                        }
+                        if (clientAuth.getNetWriteTimeout() > 0) {
+                            writeTimeout = clientAuth.getNetWriteTimeout();
+                        }
+                        IdleStateHandler idleStateHandler = new IdleStateHandler(NettyUtils.hashedWheelTimer,
+                            readTimeout,
+                            writeTimeout,
+                            0);
+                        ctx.getPipeline().addBefore(SessionHandler.class.getName(),
+                            IdleStateHandler.class.getName(),
+                            idleStateHandler);
+
+                        IdleStateAwareChannelHandler idleStateAwareChannelHandler = new IdleStateAwareChannelHandler() {
+
+                            public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) throws Exception {
+                                logger.warn("channel:{} idle timeout exceeds, close channel to save server resources...",
+                                    ctx.getChannel());
+                                ctx.getChannel().close();
+                            }
+
+                        };
+                        ctx.getPipeline().addBefore(SessionHandler.class.getName(),
+                            IdleStateAwareChannelHandler.class.getName(),
+                            idleStateAwareChannelHandler);
+                    }
+
+                });
+                break;
+        }
+    }
+
+    public void setEmbededServer(CanalServerWithEmbeded embededServer) {
+        this.embededServer = embededServer;
+    }
+
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/FixedHeaderFrameDecoder.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/FixedHeaderFrameDecoder.java
new file mode 100644
index 00000000..32a79d57
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/FixedHeaderFrameDecoder.java
@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.server.netty.handler;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.handler.codec.replay.ReplayingDecoder;
+import org.jboss.netty.handler.codec.replay.VoidEnum;
+
+/**
+ * 解析对应的header信息
+ * 
+ * @author jianghang 2012-10-24 上午11:31:39
+ * @version 1.0.0
+ */
+public class FixedHeaderFrameDecoder extends ReplayingDecoder {
+
+    protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer, VoidEnum state)
+                                                                                                             throws Exception {
+         return buffer.readBytes(buffer.readInt());
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java
new file mode 100644
index 00000000..a2eed0ce
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java
@@ -0,0 +1,30 @@
+package com.alibaba.otter.canal.server.netty.handler;
+
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+import com.alibaba.otter.canal.server.netty.NettyUtils;
+
+/**
+ * handshake交互
+ * 
+ * @author jianghang 2012-10-24 上午11:39:54
+ * @version 1.0.0
+ */
+public class HandshakeInitializationHandler extends SimpleChannelHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(HandshakeInitializationHandler.class);
+
+    public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+        byte[] body = Packet.newBuilder().setType(CanalPacket.PacketType.HANDSHAKE).setBody(
+                                                                                            Handshake.newBuilder().build().toByteString()).build().toByteArray();
+        NettyUtils.write(ctx.getChannel(), body, null);
+        logger.info("send handshake initialization packet to : {}", ctx.getChannel());
+    }
+}
diff --git a/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
new file mode 100644
index 00000000..54ccd58b
--- /dev/null
+++ b/server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
@@ -0,0 +1,249 @@
+package com.alibaba.otter.canal.server.netty.handler;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.slf4j.helpers.MessageFormatter;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalPacket;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
+import com.alibaba.otter.canal.protocol.CanalPacket.Get;
+import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
+import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.netty.NettyUtils;
+
+/**
+ * 处理具体的客户端请求
+ * 
+ * @author jianghang 2012-10-24 下午02:21:13
+ * @version 1.0.0
+ */
+public class SessionHandler extends SimpleChannelHandler {
+
+    private static final Logger    logger = LoggerFactory.getLogger(SessionHandler.class);
+    private CanalServerWithEmbeded embededServer;
+
+    public SessionHandler(){
+
+    }
+
+    public SessionHandler(CanalServerWithEmbeded embededServer){
+        this.embededServer = embededServer;
+    }
+
+    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+        logger.info("message receives in session handler...");
+        ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
+        Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
+        ClientIdentity clientIdentity = null;
+        try {
+            switch (packet.getType()) {
+                case SUBSCRIPTION:
+                    Sub sub = Sub.parseFrom(packet.getBody());
+                    if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
+                        clientIdentity = new ClientIdentity(sub.getDestination(), Short.valueOf(sub.getClientId()),
+                                                            sub.getFilter());
+                        MDC.put("destination", clientIdentity.getDestination());
+                        embededServer.subscribe(clientIdentity);
+
+                        // 尝试启动，如果已经启动，忽略
+                        if (!embededServer.isStart(clientIdentity.getDestination())) {
+                            ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
+                            if (!runningMonitor.isStart()) {
+                                runningMonitor.start();
+                            }
+                        }
+
+                        ctx.setAttachment(clientIdentity);// 设置状态数据
+                        NettyUtils.ack(ctx.getChannel(), null);
+                    } else {
+                        NettyUtils.error(401,
+                                         MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage(),
+                                         ctx.getChannel(), null);
+                    }
+                    break;
+                case UNSUBSCRIPTION:
+                    Unsub unsub = Unsub.parseFrom(packet.getBody());
+                    if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
+                        clientIdentity = new ClientIdentity(unsub.getDestination(), Short.valueOf(unsub.getClientId()),
+                                                            unsub.getFilter());
+                        MDC.put("destination", clientIdentity.getDestination());
+                        embededServer.unsubscribe(clientIdentity);
+                        stopCanalInstanceIfNecessary(clientIdentity);// 尝试关闭
+                        NettyUtils.ack(ctx.getChannel(), null);
+                    } else {
+                        NettyUtils.error(401,
+                                         MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage(),
+                                         ctx.getChannel(), null);
+                    }
+                    break;
+                case GET:
+                    Get get = CanalPacket.Get.parseFrom(packet.getBody());
+                    if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
+                        clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
+                        MDC.put("destination", clientIdentity.getDestination());
+                        Message message = null;
+
+                        //                        if (get.getAutoAck()) {
+                        //                            if (get.getTimeout() == -1) {//是否是初始值
+                        //                                message = embededServer.get(clientIdentity, get.getFetchSize());
+                        //                            } else {
+                        //                                TimeUnit unit = convertTimeUnit(get.getUnit());
+                        //                                message = embededServer.get(clientIdentity, get.getFetchSize(), get.getTimeout(), unit);
+                        //                            }
+                        //                        } else {
+                        if (get.getTimeout() == -1) {//是否是初始值
+                            message = embededServer.getWithoutAck(clientIdentity, get.getFetchSize());
+                        } else {
+                            TimeUnit unit = convertTimeUnit(get.getUnit());
+                            message = embededServer.getWithoutAck(clientIdentity, get.getFetchSize(), get.getTimeout(),
+                                                                  unit);
+                        }
+                        //                        }
+
+                        Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
+                        packetBuilder.setType(PacketType.MESSAGES);
+
+                        Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
+                        messageBuilder.setBatchId(message.getId());
+                        if (message.getId() != -1 && !CollectionUtils.isEmpty(message.getEntries())) {
+                            for (Entry entry : message.getEntries()) {
+                                messageBuilder.addMessages(entry.toByteString());
+                            }
+                        }
+                        packetBuilder.setBody(messageBuilder.build().toByteString());
+                        NettyUtils.write(ctx.getChannel(), packetBuilder.build().toByteArray(), null);// 输出数据
+                    } else {
+                        NettyUtils.error(401,
+                                         MessageFormatter.format("destination or clientId is null", get.toString()).getMessage(),
+                                         ctx.getChannel(), null);
+                    }
+                    break;
+                case CLIENTACK:
+                    ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
+                    MDC.put("destination", ack.getDestination());
+                    if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
+                        if (ack.getBatchId() == 0L) {
+                            NettyUtils.error(402,
+                                             MessageFormatter.format("batchId should assign value", ack.toString()).getMessage(),
+                                             ctx.getChannel(), null);
+                        } else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据，直接忽略之
+                            // donothing
+                        } else {
+                            clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
+                            embededServer.ack(clientIdentity, ack.getBatchId());
+                        }
+                    } else {
+                        NettyUtils.error(401,
+                                         MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage(),
+                                         ctx.getChannel(), null);
+                    }
+                    break;
+                case CLIENTROLLBACK:
+                    ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
+                    MDC.put("destination", rollback.getDestination());
+                    if (StringUtils.isNotEmpty(rollback.getDestination())
+                        && StringUtils.isNotEmpty(rollback.getClientId())) {
+                        clientIdentity = new ClientIdentity(rollback.getDestination(),
+                                                            Short.valueOf(rollback.getClientId()));
+                        if (rollback.getBatchId() == 0L) {
+                            embededServer.rollback(clientIdentity);// 回滚所有批次
+                        } else {
+                            embededServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次
+                        }
+                    } else {
+                        NettyUtils.error(401,
+                                         MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage(),
+                                         ctx.getChannel(), null);
+                    }
+                    break;
+                default:
+                    NettyUtils.error(400,
+                                     MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage(),
+                                     ctx.getChannel(), null);
+                    break;
+            }
+        } catch (Throwable exception) {
+            NettyUtils.error(400,
+                             MessageFormatter.format("something goes wrong with channel:{}, exception={}",
+                                                     ctx.getChannel(), ExceptionUtils.getStackTrace(exception)).getMessage(),
+                             ctx.getChannel(), null);
+        } finally {
+            MDC.remove("destination");
+        }
+    }
+
+    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+        logger.error("something goes wrong with channel:{}, exception={}", ctx.getChannel(),
+                     ExceptionUtils.getStackTrace(e.getCause()));
+
+        ctx.getChannel().close();
+    }
+
+    public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+        // logger.info("remove binding subscription value object if any...");
+        // ClientIdentity clientIdentity = (ClientIdentity) ctx.getAttachment();
+        // // 如果唯一的订阅者都取消了订阅，直接关闭服务，针对内部版本模式下可以减少资源浪费
+        // if (clientIdentity != null) {
+        // stopCanalInstanceIfNecessary(clientIdentity);
+        // }
+    }
+
+    private void stopCanalInstanceIfNecessary(ClientIdentity clientIdentity) {
+        List clientIdentitys = embededServer.listAllSubscribe(clientIdentity.getDestination());
+        if (clientIdentitys != null && clientIdentitys.size() == 1 && clientIdentitys.contains(clientIdentity)) {
+            ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
+            if (runningMonitor.isStart()) {
+                runningMonitor.release();
+            }
+        }
+    }
+
+    private TimeUnit convertTimeUnit(int unit) {
+        switch (unit) {
+            case 0:
+                return TimeUnit.NANOSECONDS;
+            case 1:
+                return TimeUnit.MICROSECONDS;
+            case 2:
+                return TimeUnit.MILLISECONDS;
+            case 3:
+                return TimeUnit.SECONDS;
+            case 4:
+                return TimeUnit.MINUTES;
+            case 5:
+                return TimeUnit.HOURS;
+            case 6:
+                return TimeUnit.DAYS;
+            default:
+                return TimeUnit.MILLISECONDS;
+        }
+    }
+
+    public void setEmbededServer(CanalServerWithEmbeded embededServer) {
+        this.embededServer = embededServer;
+    }
+
+}
diff --git a/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java b/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java
new file mode 100644
index 00000000..b0dad135
--- /dev/null
+++ b/server/src/test/java/com/alibaba/otter/canal/server/BaseCanalServerWithEmbededTest.java
@@ -0,0 +1,197 @@
+package com.alibaba.otter.canal.server;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager;
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+import com.alibaba.otter.canal.parse.CanalEventParser;
+import com.alibaba.otter.canal.parse.CanalHASwitchable;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+
+public abstract class BaseCanalServerWithEmbededTest {
+
+    protected static final String  cluster1       = "127.0.0.1:2188";
+    protected static final String  DESTINATION    = "ljhtest1";
+    protected static final String  DETECTING_SQL  = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+    protected static final String  MYSQL_ADDRESS  = "127.0.0.1";
+    protected static final String  USERNAME       = "retl";
+    protected static final String  PASSWORD       = "retl";
+    protected static final String  FILTER         = "retl\\..*,erosa.zk_complaint_bizdata";
+
+    private CanalServerWithEmbeded server;
+    private ClientIdentity         clientIdentity = new ClientIdentity(DESTINATION, (short) 1);                               ;
+
+    @Before
+    public void setUp() {
+        server = new CanalServerWithEmbeded();
+        server.setCanalInstanceGenerator(new CanalInstanceGenerator() {
+
+            public CanalInstance generate(String destination) {
+                Canal canal = buildCanal();
+                return new CanalInstanceWithManager(canal, FILTER);
+            }
+        });
+        server.start();
+        server.start(DESTINATION);
+    }
+
+    @After
+    public void tearDown() {
+        server.stop();
+    }
+
+    @Test
+    public void testGetWithoutAck() {
+        int maxEmptyCount = 10;
+        int emptyCount = 0;
+        int totalCount = 0;
+        server.subscribe(clientIdentity);
+        while (emptyCount < maxEmptyCount) {
+            Message message = server.getWithoutAck(clientIdentity, 11);
+            if (CollectionUtils.isEmpty(message.getEntries())) {
+                emptyCount++;
+                try {
+                    Thread.sleep(emptyCount * 300L);
+                } catch (InterruptedException e) {
+                    Assert.fail();
+                }
+
+                System.out.println("empty count : " + emptyCount);
+            } else {
+                emptyCount = 0;
+                totalCount += message.getEntries().size();
+                server.ack(clientIdentity, message.getId());
+            }
+        }
+
+        System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
+        server.unsubscribe(clientIdentity);
+    }
+
+    @Test
+    public void testGet() {
+        int maxEmptyCount = 10;
+        int emptyCount = 0;
+        int totalCount = 0;
+        server.subscribe(clientIdentity);
+        while (emptyCount < maxEmptyCount) {
+            Message message = server.get(clientIdentity, 11);
+            if (CollectionUtils.isEmpty(message.getEntries())) {
+                emptyCount++;
+                try {
+                    Thread.sleep(emptyCount * 300L);
+                } catch (InterruptedException e) {
+                    Assert.fail();
+                }
+
+                System.out.println("empty count : " + emptyCount);
+            } else {
+                emptyCount = 0;
+                totalCount += message.getEntries().size();
+            }
+        }
+
+        System.out.println("!!!!!! testGet totalCount : " + totalCount);
+        server.unsubscribe(clientIdentity);
+    }
+
+    // @Test
+    public void testRollback() {
+        int maxEmptyCount = 10;
+        int emptyCount = 0;
+        int totalCount = 0;
+        server.subscribe(clientIdentity);
+        while (emptyCount < maxEmptyCount) {
+            Message message = server.getWithoutAck(clientIdentity, 11);
+            if (CollectionUtils.isEmpty(message.getEntries())) {
+                emptyCount++;
+                try {
+                    Thread.sleep(emptyCount * 300L);
+                } catch (InterruptedException e) {
+                    Assert.fail();
+                }
+
+                System.out.println("empty count : " + emptyCount);
+            } else {
+                emptyCount = 0;
+                totalCount += message.getEntries().size();
+            }
+        }
+        System.out.println("!!!!!! testRollback totalCount : " + totalCount);
+
+        server.rollback(clientIdentity);// 直接rollback掉，再取一次
+        emptyCount = 0;
+        totalCount = 0;
+        while (emptyCount < maxEmptyCount) {
+            Message message = server.getWithoutAck(clientIdentity, 11);
+            if (CollectionUtils.isEmpty(message.getEntries())) {
+                emptyCount++;
+                try {
+                    Thread.sleep(emptyCount * 300L);
+                } catch (InterruptedException e) {
+                    Assert.fail();
+                }
+
+                System.out.println("empty count : " + emptyCount);
+            } else {
+                emptyCount = 0;
+                totalCount += message.getEntries().size();
+            }
+        }
+
+        System.out.println("!!!!!! testRollback after rollback ,  totalCount : " + totalCount);
+        server.unsubscribe(clientIdentity);
+    }
+
+    // @Test
+    public void testSwitch() {
+        int maxEmptyCount = 10;
+        int emptyCount = 0;
+        int totalCount = 0;
+
+        int thresold = 50;
+        int batchSize = 11;
+        server.subscribe(clientIdentity);
+        while (emptyCount < maxEmptyCount) {
+            Message message = server.get(clientIdentity, batchSize);
+            if (CollectionUtils.isEmpty(message.getEntries())) {
+                emptyCount++;
+                try {
+                    Thread.sleep(emptyCount * 300L);
+                } catch (InterruptedException e) {
+                    Assert.fail();
+                }
+
+                System.out.println("empty count : " + emptyCount);
+            } else {
+                emptyCount = 0;
+                totalCount += message.getEntries().size();
+
+                if ((totalCount + 1) % 100 >= thresold && (totalCount + 1) % 100 <= thresold + batchSize) {
+                    CanalEventParser eventParser = server.getCanalInstances().get(DESTINATION).getEventParser();
+                    if (eventParser instanceof CanalHASwitchable) {
+                        ((CanalHASwitchable) eventParser).doSwitch();// 执行切换
+                        try {
+                            Thread.sleep(5 * 1000); // 等待parser启动
+                        } catch (InterruptedException e) {
+                            Assert.fail();
+                        }
+                    }
+                }
+            }
+        }
+
+        System.out.println("!!!!!! testGet totalCount : " + totalCount);
+        server.unsubscribe(clientIdentity);
+    }
+
+    abstract protected Canal buildCanal();
+}
diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandaloneTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandaloneTest.java
new file mode 100644
index 00000000..9b14d32e
--- /dev/null
+++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandaloneTest.java
@@ -0,0 +1,56 @@
+package com.alibaba.otter.canal.server;
+
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
+
+public class CanalServerWithEmbeded_StandaloneTest extends BaseCanalServerWithEmbededTest {
+
+    protected Canal buildCanal() {
+        Canal canal = new Canal();
+        canal.setId(1L);
+        canal.setName(DESTINATION);
+        canal.setDesc("test");
+
+        CanalParameter parameter = new CanalParameter();
+
+        parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
+        parameter.setMetaMode(MetaMode.MEMORY);
+        parameter.setHaMode(HAMode.HEARTBEAT);
+        parameter.setIndexMode(IndexMode.MEMORY);
+
+        parameter.setStorageMode(StorageMode.MEMORY);
+        parameter.setMemoryStorageBufferSize(32 * 1024);
+
+        parameter.setSourcingType(SourcingType.MYSQL);
+        parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
+            new InetSocketAddress(MYSQL_ADDRESS, 3306)));
+        parameter.setDbUsername(USERNAME);
+        parameter.setDbPassword(PASSWORD);
+        parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
+            "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));
+
+        parameter.setSlaveId(1234L);
+
+        parameter.setDefaultConnectionTimeoutInSeconds(30);
+        parameter.setConnectionCharset("UTF-8");
+        parameter.setConnectionCharsetNumber((byte) 33);
+        parameter.setReceiveBufferSize(8 * 1024);
+        parameter.setSendBufferSize(8 * 1024);
+
+        parameter.setDetectingEnable(false);
+        parameter.setDetectingIntervalInSeconds(10);
+        parameter.setDetectingRetryTimes(3);
+        parameter.setDetectingSQL(DETECTING_SQL);
+
+        canal.setCanalParameter(parameter);
+        return canal;
+    }
+}
diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandbyTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandbyTest.java
new file mode 100644
index 00000000..e20ce69d
--- /dev/null
+++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithEmbeded_StandbyTest.java
@@ -0,0 +1,68 @@
+package com.alibaba.otter.canal.server;
+
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+
+import org.I0Itec.zkclient.ZkClient;
+import org.junit.Before;
+
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
+
+public class CanalServerWithEmbeded_StandbyTest extends BaseCanalServerWithEmbededTest {
+
+    private ZkClient zkClient = new ZkClient(cluster1);
+
+    @Before
+    public void setUp() {
+        zkClient.deleteRecursive(ZookeeperPathUtils.CANAL_ROOT_NODE);
+        super.setUp();
+    }
+
+    protected Canal buildCanal() {
+        Canal canal = new Canal();
+        canal.setId(1L);
+        canal.setName(DESTINATION);
+        canal.setDesc("test");
+
+        CanalParameter parameter = new CanalParameter();
+
+        parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
+        parameter.setMetaMode(MetaMode.MIXED); // 冷备，可选择混合模式
+        parameter.setHaMode(HAMode.HEARTBEAT);
+        parameter.setIndexMode(IndexMode.META);// 内存版store，需要选择meta做为index
+
+        parameter.setStorageMode(StorageMode.MEMORY);
+        parameter.setMemoryStorageBufferSize(32 * 1024);
+
+        parameter.setSourcingType(SourcingType.MYSQL);
+        parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
+            new InetSocketAddress(MYSQL_ADDRESS, 3306)));
+        parameter.setDbUsername(USERNAME);
+        parameter.setDbPassword(PASSWORD);
+        parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
+            "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));
+
+        parameter.setSlaveId(1234L);
+
+        parameter.setDefaultConnectionTimeoutInSeconds(30);
+        parameter.setConnectionCharset("UTF-8");
+        parameter.setConnectionCharsetNumber((byte) 33);
+        parameter.setReceiveBufferSize(8 * 1024);
+        parameter.setSendBufferSize(8 * 1024);
+
+        parameter.setDetectingEnable(false);
+        parameter.setDetectingIntervalInSeconds(10);
+        parameter.setDetectingRetryTimes(3);
+        parameter.setDetectingSQL(DETECTING_SQL);
+
+        canal.setCanalParameter(parameter);
+        return canal;
+    }
+}
diff --git a/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java
new file mode 100644
index 00000000..dddad6b6
--- /dev/null
+++ b/server/src/test/java/com/alibaba/otter/canal/server/CanalServerWithNettyTest.java
@@ -0,0 +1,263 @@
+package com.alibaba.otter.canal.server;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+import java.util.Arrays;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager;
+import com.alibaba.otter.canal.instance.manager.model.Canal;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType;
+import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode;
+import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
+import com.alibaba.otter.canal.protocol.CanalPacket.Get;
+import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
+import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
+import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
+import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
+
+public class CanalServerWithNettyTest {
+
+    protected static final String cluster1      = "127.0.0.1:2188";
+    protected static final String DESTINATION   = "ljhtest1";
+    protected static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
+    protected static final String MYSQL_ADDRESS = "127.0.0.1";
+    protected static final String USERNAME      = "retl";
+    protected static final String PASSWORD      = "retl";
+    protected static final String FILTER        = "retl\\..*,erosa.canaltable1s,erosa.canaltable1t";
+
+    private final ByteBuffer      header        = ByteBuffer.allocate(4);
+    private CanalServerWithNetty  nettyServer;
+
+    @Before
+    public void setUp() {
+        CanalServerWithEmbeded embededServer = new CanalServerWithEmbeded();
+        embededServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {
+
+            public CanalInstance generate(String destination) {
+                Canal canal = buildCanal();
+                return new CanalInstanceWithManager(canal, FILTER);
+            }
+        });
+
+        nettyServer = new CanalServerWithNetty(embededServer);
+        nettyServer.setPort(1088);
+        nettyServer.start();
+    }
+
+    @Test
+    public void testAuth() {
+        try {
+            SocketChannel channel = SocketChannel.open();
+            channel.connect(new InetSocketAddress("127.0.0.1", 1088));
+            Packet p = Packet.parseFrom(readNextPacket(channel));
+
+            if (p.getVersion() != 1) {
+                throw new Exception("unsupported version at this client.");
+            }
+
+            if (p.getType() != PacketType.HANDSHAKE) {
+                throw new Exception("expect handshake but found other type.");
+            }
+            //
+            Handshake handshake = Handshake.parseFrom(p.getBody());
+            System.out.println(handshake.getSupportedCompressionsList());
+            //
+            ClientAuth ca = ClientAuth.newBuilder()
+                .setUsername("")
+                .setNetReadTimeout(10000)
+                .setNetWriteTimeout(10000)
+                .build();
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.CLIENTAUTHENTICATION)
+                    .setBody(ca.toByteString())
+                    .build()
+                    .toByteArray());
+            //
+            p = Packet.parseFrom(readNextPacket(channel));
+            if (p.getType() != PacketType.ACK) {
+                throw new Exception("unexpected packet type when ack is expected");
+            }
+
+            Ack ack = Ack.parseFrom(p.getBody());
+            if (ack.getErrorCode() > 0) {
+                throw new Exception("something goes wrong when doing authentication: " + ack.getErrorMessage());
+            }
+
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.SUBSCRIPTION)
+                    .setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
+                    .build()
+                    .toByteArray());
+            //
+            p = Packet.parseFrom(readNextPacket(channel));
+            ack = Ack.parseFrom(p.getBody());
+            if (ack.getErrorCode() > 0) {
+                throw new Exception("failed to subscribe with reason: " + ack.getErrorMessage());
+            }
+
+            for (int i = 0; i < 10; i++) {
+                writeWithHeader(channel,
+                    Packet.newBuilder()
+                        .setType(PacketType.GET)
+                        .setBody(Get.newBuilder()
+                            .setDestination(DESTINATION)
+                            .setClientId("1")
+                            .setFetchSize(10)
+                            .build()
+                            .toByteString())
+                        .build()
+                        .toByteArray());
+                p = Packet.parseFrom(readNextPacket(channel));
+
+                long batchId = -1L;
+                switch (p.getType()) {
+                    case MESSAGES: {
+                        Messages messages = Messages.parseFrom(p.getBody());
+                        batchId = messages.getBatchId();
+                        break;
+                    }
+                    case ACK: {
+                        ack = Ack.parseFrom(p.getBody());
+                        if (ack.getErrorCode() > 0) {
+                            throw new Exception("failed to subscribe with reason: " + ack.getErrorMessage());
+                        }
+                        break;
+                    }
+                    default: {
+                        throw new Exception("unexpected packet type: " + p.getType());
+                    }
+                }
+
+                System.out.println("!!!!!!!!!!!!!!!!! " + batchId);
+                Thread.sleep(1000L);
+                writeWithHeader(channel,
+                    Packet.newBuilder()
+                        .setType(PacketType.CLIENTACK)
+                        .setBody(ClientAck.newBuilder()
+                            .setDestination(DESTINATION)
+                            .setClientId("1")
+                            .setBatchId(batchId)
+                            .build()
+                            .toByteString())
+                        .build()
+                        .toByteArray());
+            }
+
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.CLIENTROLLBACK)
+                    .setBody(ClientRollback.newBuilder()
+                        .setDestination(DESTINATION)
+                        .setClientId("1")
+                        .build()
+                        .toByteString())
+                    .build()
+                    .toByteArray());
+
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.UNSUBSCRIPTION)
+                    .setBody(Unsub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
+                    .build()
+                    .toByteArray());
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @After
+    public void tearDown() {
+        nettyServer.stop();
+    }
+
+    private byte[] readNextPacket(SocketChannel channel) throws IOException {
+        header.clear();
+        read(channel, header);
+        int bodyLen = header.getInt(0);
+        ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen);
+        read(channel, bodyBuf);
+        return bodyBuf.array();
+    }
+
+    private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
+        ByteBuffer header = ByteBuffer.allocate(4);
+        header.putInt(body.length);
+        header.flip();
+        int len = channel.write(header);
+        assert (len == header.capacity());
+
+        channel.write(ByteBuffer.wrap(body));
+    }
+
+    private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
+        while (buffer.hasRemaining()) {
+            int r = channel.read(buffer);
+            if (r == -1) {
+                throw new IOException("end of stream when reading header");
+            }
+        }
+    }
+
+    private Canal buildCanal() {
+        Canal canal = new Canal();
+        canal.setId(1L);
+        canal.setName(DESTINATION);
+        canal.setDesc("test");
+
+        CanalParameter parameter = new CanalParameter();
+
+        parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
+        parameter.setMetaMode(MetaMode.MEMORY);
+        parameter.setHaMode(HAMode.HEARTBEAT);
+        parameter.setIndexMode(IndexMode.MEMORY);
+
+        parameter.setStorageMode(StorageMode.MEMORY);
+        parameter.setMemoryStorageBufferSize(32 * 1024);
+
+        parameter.setSourcingType(SourcingType.MYSQL);
+        parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
+            new InetSocketAddress(MYSQL_ADDRESS, 3306)));
+        parameter.setDbUsername(USERNAME);
+        parameter.setDbPassword(PASSWORD);
+        parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
+            "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));
+
+        parameter.setSlaveId(1234L);
+
+        parameter.setDefaultConnectionTimeoutInSeconds(30);
+        parameter.setConnectionCharset("UTF-8");
+        parameter.setConnectionCharsetNumber((byte) 33);
+        parameter.setReceiveBufferSize(8 * 1024);
+        parameter.setSendBufferSize(8 * 1024);
+
+        parameter.setDetectingEnable(false);
+        parameter.setDetectingIntervalInSeconds(10);
+        parameter.setDetectingRetryTimes(3);
+        parameter.setDetectingSQL(DETECTING_SQL);
+
+        canal.setCanalParameter(parameter);
+        return canal;
+    }
+}
diff --git a/sink/pom.xml b/sink/pom.xml
new file mode 100644
index 00000000..c0b4d3fc
--- /dev/null
+++ b/sink/pom.xml
@@ -0,0 +1,41 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.sink
+	jar
+	canal sink module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.protocol
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.filter
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.store
+			${project.version}
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventDownStreamHandler.java b/sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventDownStreamHandler.java
new file mode 100644
index 00000000..c7593348
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventDownStreamHandler.java
@@ -0,0 +1,25 @@
+package com.alibaba.otter.canal.sink;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+
+/**
+ * 默认的实现
+ * 
+ * @author jianghang 2013-10-8 下午8:35:29
+ * @since 1.0.12
+ */
+public class AbstractCanalEventDownStreamHandler extends AbstractCanalLifeCycle implements CanalEventDownStreamHandler {
+
+    public T before(T events) {
+        return events;
+    }
+
+    public T retry(T events) {
+        return events;
+    }
+
+    public T after(T events) {
+        return events;
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventSink.java
new file mode 100644
index 00000000..db23cbfe
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/AbstractCanalEventSink.java
@@ -0,0 +1,53 @@
+package com.alibaba.otter.canal.sink;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.filter.CanalEventFilter;
+
+/**
+ * @author jianghang 2012-7-23 下午01:02:45
+ */
+public abstract class AbstractCanalEventSink extends AbstractCanalLifeCycle implements CanalEventSink {
+
+    protected CanalEventFilter                  filter;
+    protected List handlers = new ArrayList();
+
+    public void setFilter(CanalEventFilter filter) {
+        this.filter = filter;
+    }
+
+    public void addHandler(CanalEventDownStreamHandler handler) {
+        this.handlers.add(handler);
+    }
+
+    public CanalEventDownStreamHandler getHandler(int index) {
+        return this.handlers.get(index);
+    }
+
+    public void addHandler(CanalEventDownStreamHandler handler, int index) {
+        this.handlers.add(index, handler);
+    }
+
+    public void removeHandler(int index) {
+        this.handlers.remove(index);
+    }
+
+    public void removeHandler(CanalEventDownStreamHandler handler) {
+        this.handlers.remove(handler);
+    }
+
+    public CanalEventFilter getFilter() {
+        return filter;
+    }
+
+    public List getHandlers() {
+        return handlers;
+    }
+
+    public void interrupt() {
+        // do nothing
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventDownStreamHandler.java b/sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventDownStreamHandler.java
new file mode 100644
index 00000000..0ded49aa
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventDownStreamHandler.java
@@ -0,0 +1,27 @@
+package com.alibaba.otter.canal.sink;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+
+/**
+ * 处理下sink时的数据流
+ * 
+ * @author jianghang 2012-7-31 下午03:06:26
+ * @version 1.0.0
+ */
+public interface CanalEventDownStreamHandler extends CanalLifeCycle {
+
+    /**
+     * 提交到store之前做一下处理，允许替换Event
+     */
+    public T before(T events);
+
+    /**
+     * store处于full后，retry时处理做一下处理
+     */
+    public T retry(T events);
+
+    /**
+     * 提交store成功后做一下处理
+     */
+    public T after(T events);
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventSink.java
new file mode 100644
index 00000000..d0ca8838
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/CanalEventSink.java
@@ -0,0 +1,38 @@
+package com.alibaba.otter.canal.sink;
+
+import java.net.InetSocketAddress;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+
+/**
+ * event事件消费者
+ * 
+ * 
+ * 1. 剥离filter/sink为独立的两个动作，方便在快速判断数据是否有效
+ * 

+ * 
+ * @author jianghang 2012-6-21 下午05:03:40
+ * @version 1.0.0
+ */
+public interface CanalEventSink extends CanalLifeCycle {
+
+    /**
+     * 提交数据
+     * 
+     * @param event
+     * @param remoteAddress
+     * @param destination
+     * @throws CanalSinkException
+     * @throws InterruptedException
+     */
+    boolean sink(T event, InetSocketAddress remoteAddress, String destination) throws CanalSinkException,
+                                                                              InterruptedException;
+
+    /**
+     * 中断消费，比如解析模块发生了切换，想临时中断当前的merge请求，清理对应的上下文状态，可见{@linkplain GroupEventSink}
+     */
+    void interrupt();
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java
new file mode 100644
index 00000000..8474950e
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/EntryEventSink.java
@@ -0,0 +1,206 @@
+package com.alibaba.otter.canal.sink.entry;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.LockSupport;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.sink.AbstractCanalEventSink;
+import com.alibaba.otter.canal.sink.CanalEventDownStreamHandler;
+import com.alibaba.otter.canal.sink.CanalEventSink;
+import com.alibaba.otter.canal.sink.exception.CanalSinkException;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * mysql binlog数据对象输出
+ * 
+ * @author jianghang 2012-7-4 下午03:23:16
+ * @version 1.0.0
+ */
+public class EntryEventSink extends AbstractCanalEventSink> implements CanalEventSink> {
+
+    private static final Logger    logger                        = LoggerFactory.getLogger(EntryEventSink.class);
+    private static final int       maxFullTimes                  = 10;
+    private CanalEventStore eventStore;
+    protected boolean              filterTransactionEntry        = false;                                        // 是否需要过滤事务头/尾
+    protected boolean              filterEmtryTransactionEntry   = true;                                         // 是否需要过滤空的事务头/尾
+    protected long                 emptyTransactionInterval      = 5 * 1000;                                     // 空的事务输出的频率
+    protected long                 emptyTransctionThresold       = 8192;                                         // 超过1024个事务头，输出一个
+    protected volatile long        lastEmptyTransactionTimestamp = 0L;
+    protected AtomicLong           lastEmptyTransactionCount     = new AtomicLong(0L);
+
+    public EntryEventSink(){
+        addHandler(new HeartBeatEntryEventHandler());
+    }
+
+    public void start() {
+        super.start();
+        Assert.notNull(eventStore);
+
+        for (CanalEventDownStreamHandler handler : getHandlers()) {
+            if (!handler.isStart()) {
+                handler.start();
+            }
+        }
+    }
+
+    public void stop() {
+        super.stop();
+
+        for (CanalEventDownStreamHandler handler : getHandlers()) {
+            if (handler.isStart()) {
+                handler.stop();
+            }
+        }
+    }
+
+    public boolean filter(List event, InetSocketAddress remoteAddress, String destination) {
+
+        return false;
+    }
+
+    public boolean sink(List entrys, InetSocketAddress remoteAddress, String destination)
+                                                                                                           throws CanalSinkException,
+                                                                                                           InterruptedException {
+        List rowDatas = entrys;
+        if (filterTransactionEntry) {
+            rowDatas = new ArrayList();
+            for (CanalEntry.Entry entry : entrys) {
+                if (entry.getEntryType() == EntryType.ROWDATA) {
+                    rowDatas.add(entry);
+                }
+            }
+        }
+
+        return sinkData(rowDatas, remoteAddress);
+    }
+
+    private boolean sinkData(List entrys, InetSocketAddress remoteAddress)
+                                                                                            throws InterruptedException {
+        boolean hasRowData = false;
+        boolean hasHeartBeat = false;
+        List events = new ArrayList();
+        for (CanalEntry.Entry entry : entrys) {
+            Event event = new Event(new LogIdentity(remoteAddress, -1L), entry);
+            if (!doFilter(event)) {
+                continue;
+            }
+
+            events.add(event);
+            hasRowData |= (entry.getEntryType() == EntryType.ROWDATA);
+            hasHeartBeat |= (entry.getEntryType() == EntryType.HEARTBEAT);
+        }
+
+        if (hasRowData) {
+            // 存在row记录
+            return doSink(events);
+        } else if (hasHeartBeat) {
+            // 存在heartbeat记录，直接跳给后续处理
+            return doSink(events);
+        } else {
+            // 需要过滤的数据
+            if (filterEmtryTransactionEntry && !CollectionUtils.isEmpty(events)) {
+                long currentTimestamp = events.get(0).getEntry().getHeader().getExecuteTime();
+                // 基于一定的策略控制，放过空的事务头和尾，便于及时更新数据库位点，表明工作正常
+                if (Math.abs(currentTimestamp - lastEmptyTransactionTimestamp) > emptyTransactionInterval
+                    || lastEmptyTransactionCount.incrementAndGet() > emptyTransctionThresold) {
+                    lastEmptyTransactionCount.set(0L);
+                    lastEmptyTransactionTimestamp = currentTimestamp;
+                    return doSink(events);
+                }
+            }
+
+            // 直接返回true，忽略空的事务头和尾
+            return true;
+        }
+    }
+
+    protected boolean doFilter(Event event) {
+        if (filter != null && event.getEntry().getEntryType() == EntryType.ROWDATA) {
+            String name = getSchemaNameAndTableName(event.getEntry());
+            boolean need = filter.filter(name);
+            if (!need) {
+                logger.debug("filter name[{}] entry : {}:{}",
+                    new Object[] { name, event.getEntry().getHeader().getLogfileName(),
+                            event.getEntry().getHeader().getLogfileOffset() });
+            }
+
+            return need;
+        } else {
+            return true;
+        }
+    }
+
+    protected boolean doSink(List events) {
+        for (CanalEventDownStreamHandler> handler : getHandlers()) {
+            events = handler.before(events);
+        }
+
+        int fullTimes = 0;
+        do {
+            if (eventStore.tryPut(events)) {
+                for (CanalEventDownStreamHandler> handler : getHandlers()) {
+                    events = handler.after(events);
+                }
+                return true;
+            } else {
+                applyWait(++fullTimes);
+            }
+
+            for (CanalEventDownStreamHandler> handler : getHandlers()) {
+                events = handler.retry(events);
+            }
+
+        } while (running && !Thread.interrupted());
+        return false;
+    }
+
+    // 处理无数据的情况，避免空循环挂死
+    private void applyWait(int fullTimes) {
+        int newFullTimes = fullTimes > maxFullTimes ? maxFullTimes : fullTimes;
+        if (fullTimes <= 3) { // 3次以内
+            Thread.yield();
+        } else { // 超过3次，最多只sleep 10ms
+            LockSupport.parkNanos(1000 * 1000L * newFullTimes);
+        }
+
+    }
+
+    private String getSchemaNameAndTableName(CanalEntry.Entry entry) {
+        StringBuilder result = new StringBuilder();
+        result.append(entry.getHeader().getSchemaName()).append(".").append(entry.getHeader().getTableName());
+        return result.toString();
+    }
+
+    public void setEventStore(CanalEventStore eventStore) {
+        this.eventStore = eventStore;
+    }
+
+    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
+        this.filterTransactionEntry = filterTransactionEntry;
+    }
+
+    public void setFilterEmtryTransactionEntry(boolean filterEmtryTransactionEntry) {
+        this.filterEmtryTransactionEntry = filterEmtryTransactionEntry;
+    }
+
+    public void setEmptyTransactionInterval(long emptyTransactionInterval) {
+        this.emptyTransactionInterval = emptyTransactionInterval;
+    }
+
+    public void setEmptyTransctionThresold(long emptyTransctionThresold) {
+        this.emptyTransctionThresold = emptyTransctionThresold;
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/HeartBeatEntryEventHandler.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/HeartBeatEntryEventHandler.java
new file mode 100644
index 00000000..9d7a9871
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/HeartBeatEntryEventHandler.java
@@ -0,0 +1,41 @@
+package com.alibaba.otter.canal.sink.entry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.sink.AbstractCanalEventDownStreamHandler;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 处理一下一下heartbeat数据
+ * 
+ * @author jianghang 2013-10-8 下午6:03:53
+ * @since 1.0.12
+ */
+public class HeartBeatEntryEventHandler extends AbstractCanalEventDownStreamHandler> {
+
+    public List before(List events) {
+        boolean existHeartBeat = false;
+        for (Event event : events) {
+            if (event.getEntry().getEntryType() == EntryType.HEARTBEAT) {
+                existHeartBeat = true;
+            }
+        }
+
+        if (!existHeartBeat) {
+            return events;
+        } else {
+            // 目前heartbeat和其他事件是分离的，保险一点还是做一下检查处理
+            List result = new ArrayList();
+            for (Event event : events) {
+                if (event.getEntry().getEntryType() != EntryType.HEARTBEAT) {
+                    result.add(event);
+                }
+            }
+
+            return result;
+        }
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupBarrier.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupBarrier.java
new file mode 100644
index 00000000..7ddefbcd
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupBarrier.java
@@ -0,0 +1,42 @@
+package com.alibaba.otter.canal.sink.entry.group;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * 针对group合并的barrier接口，控制多个sink操作的合并处理
+ * 
+ * @author jianghang 2012-10-18 下午05:07:35
+ * @version 1.0.0
+ */
+public interface GroupBarrier {
+
+    /**
+     * 判断当前的数据对象是否允许通过
+     * 
+     * @param event
+     * @throws InterruptedException
+     */
+    public void await(T event) throws InterruptedException;
+
+    /**
+     * 判断当前的数据对象是否允许通过，带超时控制
+     * 
+     * @param event
+     * @param timeout
+     * @param unit
+     * @throws InterruptedException
+     * @throws TimeoutException
+     */
+    public void await(T event, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException;
+
+    /**
+     * sink成功，清理对应barrier的状态
+     */
+    public void clear(T event);
+
+    /**
+     * 出现切换，发起interrupt，清理对应的上下文
+     */
+    public void interrupt();
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupEventSink.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupEventSink.java
new file mode 100644
index 00000000..71ac25e3
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/GroupEventSink.java
@@ -0,0 +1,75 @@
+package com.alibaba.otter.canal.sink.entry.group;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.alibaba.otter.canal.sink.CanalEventDownStreamHandler;
+import com.alibaba.otter.canal.sink.entry.EntryEventSink;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 基于归并排序的sink处理
+ * 
+ * 
+ * 几点设计说明：
+ * 1. 多库合并时，需要控制不满足groupSize的条件，就会阻塞其他库的合并操作.  (比如刚启动时会所有通道正常工作才开始合并，或者中间过程出现主备切换)
+ * 2. 库解析出现问题，但没有进行主备切换，此时需要通过{@linkplain CanalEventDownStreamHandler}进行定时监听合并数据的产生时间间隔 
+ *    a. 因为一旦库解析异常，就不会再sink数据，此时groupSize就会一直缺少，就会阻塞其他库的合并，也就是不会有数据写入到store中
+ * 

+ * 
+ * @author jianghang 2012-10-15 下午09:54:18
+ * @version 1.0.0
+ */
+public class GroupEventSink extends EntryEventSink {
+
+    private int          groupSize;
+    private GroupBarrier barrier;  // 归并排序需要预先知道组的大小，用于判断是否组内所有的sink都已经开始正常取数据
+
+    public GroupEventSink(){
+        this(1);
+    }
+
+    public GroupEventSink(int groupSize){
+        super();
+        this.groupSize = groupSize;
+    }
+
+    public void start() {
+        super.start();
+
+        if (filterTransactionEntry) {
+            barrier = new TimelineBarrier(groupSize);
+        } else {
+            barrier = new TimelineTransactionBarrier(groupSize);// 支持事务保留
+        }
+    }
+
+    protected boolean doSink(List events) {
+        int size = events.size();
+        for (int i = 0; i < events.size(); i++) {
+            Event event = events.get(i);
+            try {
+                barrier.await(event);// 进行timeline的归并调度处理
+                if (filterTransactionEntry) {
+                    return super.doSink(Arrays.asList(event));
+                } else if (i == size - 1) {
+                    // 针对事务数据，只有到最后一条数据都通过后，才进行sink操作，保证原子性
+                    // 同时批量sink，也要保证在最后一条数据释放状态之前写出数据，否则就有并发问题
+                    return super.doSink(events);
+                }
+            } catch (InterruptedException e) {
+                return false;
+            } finally {
+                barrier.clear(event);
+            }
+        }
+
+        return false;
+    }
+
+    public void interrupt() {
+        super.interrupt();
+        barrier.interrupt();
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineBarrier.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineBarrier.java
new file mode 100644
index 00000000..be9edfa0
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineBarrier.java
@@ -0,0 +1,145 @@
+package com.alibaba.otter.canal.sink.entry.group;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 时间归并控制
+ * 
+ * 
+ * 大致设计：
+ *  1. 多个队列都提交一个timestamp，判断出最小的一个timestamp做为通过的条件，然后唤醒<=该最小时间的线程通过
+ *  2. 只有当多个队列都提交了一个timestamp，缺少任何一个提交，都会阻塞其他队列通过。(解决当一个库启动过慢或者发生主备切换时出现延迟等问题)
+ * 
+ * 存在一个假定，认为提交的timestamp是一个顺序递增，但是在两种case下会出现时间回退
+ * a. 大事务时，事务头的时间会晚于事务当中数据的时间，相当于出现一个时间回退
+ * b. 出现主备切换，从备机上发过来的数据会回退几秒钟
+ * 
+ * 

+ * 
+ * @author jianghang 2012-10-15 下午10:01:53
+ * @version 1.0.0
+ */
+public class TimelineBarrier implements GroupBarrier {
+
+    protected int                 groupSize;
+    protected ReentrantLock       lock           = new ReentrantLock();
+    protected Condition           condition      = lock.newCondition();
+    protected volatile long       threshold;
+    protected BlockingQueue lastTimestamps = new PriorityBlockingQueue(); // 当前通道最后一次single的时间戳
+
+    public TimelineBarrier(int groupSize){
+        this.groupSize = groupSize;
+        threshold = Long.MIN_VALUE;
+    }
+
+    /**
+     * 判断自己的timestamp是否可以通过
+     * 
+     * @throws InterruptedException
+     */
+    public void await(Event event) throws InterruptedException {
+        long timestamp = getTimestamp(event);
+        try {
+            lock.lockInterruptibly();
+            single(timestamp);
+            while (isPermit(event, timestamp) == false) {
+                condition.await();
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * 判断自己的timestamp是否可以通过,带超时控制
+     * 
+     * @throws InterruptedException
+     * @throws TimeoutException
+     */
+    public void await(Event event, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
+        long timestamp = getTimestamp(event);
+        try {
+            lock.lockInterruptibly();
+            single(timestamp);
+            while (isPermit(event, timestamp) == false) {
+                condition.await(timeout, unit);
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public void clear(Event event) {
+        // 出现中断有两种可能：
+        // 1.出现主备切换，需要剔除到Timeline中的时间占位(这样合并时就会小于groupSize，不满足调度条件，直到主备切换完成后才能重新开启合并处理)
+        // 2.出现关闭操作，退出即可
+        lastTimestamps.remove(getTimestamp(event));
+    }
+
+    public void interrupt() {
+        // do nothing，没有需要清理的上下文状态
+    }
+
+    public long state() {
+        return threshold;
+    }
+
+    /**
+     * 判断是否允许通过
+     */
+    protected boolean isPermit(Event event, long state) {
+        return state <= state();
+    }
+
+    /**
+     * 通知一下
+     */
+    protected void notify(long minTimestamp) {
+        // 通知阻塞的线程恢复, 这里采用single all操作，当group中的几个时间都相同时，一次性触发通过多个
+        condition.signalAll();
+    }
+
+    /**
+     * 通知下一个minTimestamp数据出队列
+     * 
+     * @throws InterruptedException
+     */
+    private void single(long timestamp) throws InterruptedException {
+        lastTimestamps.add(timestamp);
+
+        if (timestamp < state()) {
+            // 针对mysql事务中会出现时间跳跃
+            // 例子：
+            // 2012-08-08 16:24:26 事务头
+            // 2012-08-08 16:24:24 变更记录
+            // 2012-08-08 16:24:25 变更记录
+            // 2012-08-08 16:24:26　事务尾
+
+            // 针对这种case，一旦发现timestamp有回退的情况，直接更新threshold，强制阻塞其他的操作，等待最小数据优先处理完成
+            threshold = timestamp; // 更新为最小值
+        }
+
+        if (lastTimestamps.size() >= groupSize) {// 判断队列是否需要触发
+            // 触发下一个出队列的数据
+            Long minTimestamp = this.lastTimestamps.peek();
+            if (minTimestamp != null) {
+                threshold = minTimestamp;
+                notify(minTimestamp);
+            }
+        } else {
+            threshold = Long.MIN_VALUE;// 如果不满足队列长度，需要阻塞等待
+        }
+    }
+
+    private Long getTimestamp(Event event) {
+        return event.getEntry().getHeader().getExecuteTime();
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineTransactionBarrier.java b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineTransactionBarrier.java
new file mode 100644
index 00000000..563da644
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/entry/group/TimelineTransactionBarrier.java
@@ -0,0 +1,118 @@
+package com.alibaba.otter.canal.sink.entry.group;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 相比于{@linkplain TimelineBarrier}，增加了按事务支持，会按照事务进行分库合并处理
+ * 
+ * @author jianghang 2012-10-18 下午05:18:38
+ * @version 1.0.0
+ */
+public class TimelineTransactionBarrier extends TimelineBarrier {
+
+    private ThreadLocal inTransaction = new ThreadLocal() {
+
+                                                   protected Object initialValue() {
+                                                       return false;
+                                                   }
+                                               };
+
+    /**
+     * 
+     * 几种状态：
+     * 0：初始状态，允许大家竞争
+     * 1: 事务数据处理中
+     * 2: 非事务数据处理中
+     * 

+     */
+    private AtomicInteger        txState       = new AtomicInteger(0);
+
+    public TimelineTransactionBarrier(int groupSize){
+        super(groupSize);
+    }
+
+    public void await(Event event) throws InterruptedException {
+        try {
+            super.await(event);
+        } catch (InterruptedException e) {
+            // 出现线程中断，可能是因为关闭或者主备切换
+            // 主备切换对应的事务尾会未正常发送，需要强制设置为事务结束，允许其他队列通过
+            reset();
+            throw e;
+        }
+    }
+
+    public void await(Event event, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
+        try {
+            super.await(event, timeout, unit);
+        } catch (InterruptedException e) {
+            // 出现线程中断，可能是因为关闭或者主备切换
+            // 主备切换对应的事务尾会未正常发送，需要强制设置为事务结束，允许其他队列通过
+            reset();
+            throw e;
+        }
+    }
+
+    public void clear(Event event) {
+        super.clear(event);
+
+        if (isTransactionEnd(event)) {
+            inTransaction.set(false); // 事务结束并且已经成功写入store，清理标记，进入重新排队判断，允许新的事务进入
+            txState.compareAndSet(1, 0);
+            //            if (txState.compareAndSet(1, 0) == false) {
+            //                throw new CanalSinkException("state is not correct in transaction");
+            //            }
+        } else if (txState.intValue() == 2) {//非事务中
+            txState.compareAndSet(2, 0);
+            //            if (txState.compareAndSet(2, 0) == false) {
+            //                throw new CanalSinkException("state is not correct in non-transaction");
+            //            }
+        }
+    }
+
+    protected boolean isPermit(Event event, long state) {
+        if (txState.intValue() == 1 && inTransaction.get()) { // 如果处于事务中，直接允许通过。因为事务头已经做过判断
+            return true;
+        } else if (txState.intValue() == 0) {
+            boolean result = super.isPermit(event, state);
+            if (result) {
+                // 可能第一条送过来的数据不为Begin，需要做判断处理，如果非事务，允许直接通过，比如DDL语句
+                if (isTransactionBegin(event)) {
+                    if (txState.compareAndSet(0, 1)) {
+                        inTransaction.set(true);
+                        return true; //事务允许通过
+                    }
+                } else if (txState.compareAndSet(0, 2)) { //非事务保护中
+                    return true; //DDL/DCL允许通过
+                }
+            }
+        }
+
+        return false;
+    }
+
+    public void interrupt() {
+        super.interrupt();
+        reset();
+    }
+
+    // 重新设置状态
+    private void reset() {
+        inTransaction.remove();
+        txState.set(0);//重新置位
+    }
+
+    private boolean isTransactionBegin(Event event) {
+        return event.getEntry().getEntryType() == EntryType.TRANSACTIONBEGIN;
+    }
+
+    private boolean isTransactionEnd(Event event) {
+        return event.getEntry().getEntryType() == EntryType.TRANSACTIONEND;
+    }
+
+}
diff --git a/sink/src/main/java/com/alibaba/otter/canal/sink/exception/CanalSinkException.java b/sink/src/main/java/com/alibaba/otter/canal/sink/exception/CanalSinkException.java
new file mode 100644
index 00000000..f3cbfd91
--- /dev/null
+++ b/sink/src/main/java/com/alibaba/otter/canal/sink/exception/CanalSinkException.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.sink.exception;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalSinkException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalSinkException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalSinkException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalSinkException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalSinkException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalSinkException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/sink/src/test/java/com/alibaba/otter/canal/sink/GroupEventSinkTest.java b/sink/src/test/java/com/alibaba/otter/canal/sink/GroupEventSinkTest.java
new file mode 100644
index 00000000..1a6e8f84
--- /dev/null
+++ b/sink/src/test/java/com/alibaba/otter/canal/sink/GroupEventSinkTest.java
@@ -0,0 +1,117 @@
+package com.alibaba.otter.canal.sink;
+
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.lang.math.RandomUtils;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.sink.entry.group.GroupEventSink;
+import com.alibaba.otter.canal.sink.stub.DummyEventStore;
+
+public class GroupEventSinkTest {
+
+    private final InetSocketAddress address = new InetSocketAddress("127.0.0.1", 3306);
+
+    @Test
+    public void testGroupTwo() {
+        final DummyEventStore eventStore = new DummyEventStore();
+        final GroupEventSink eventSink = new GroupEventSink(3);
+        eventSink.setFilterTransactionEntry(true);
+        eventSink.setEventStore(eventStore);
+        eventSink.start();
+
+        ExecutorService executor = Executors.newFixedThreadPool(3);
+        final CountDownLatch latch = new CountDownLatch(1);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 50; i++) {
+                    try {
+                        eventSink.sink(Arrays.asList(buildEntry("1", 1L + i, 1L + i)), address, "ljhtest1");
+                        Thread.sleep(50L + RandomUtils.nextInt(50));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+
+                for (int i = 0; i < 50; i++) {
+                    try {
+                        eventSink.sink(Arrays.asList(buildEntry("1", 1L + i, 30L + i)), address, "ljhtest1");
+                        Thread.sleep(50L + RandomUtils.nextInt(50));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+
+                System.out.println("one sink finished!");
+                latch.countDown();
+            }
+        });
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 50; i++) {
+                    try {
+                        eventSink.sink(Arrays.asList(buildEntry("1", 1L + i, 10L + i)), address, "ljhtest2");
+                        Thread.sleep(50L + RandomUtils.nextInt(50));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+
+                for (int i = 0; i < 50; i++) {
+                    try {
+                        eventSink.sink(Arrays.asList(buildEntry("1", 1L + i, 40L + i)), address, "ljhtest2");
+                        Thread.sleep(50L + RandomUtils.nextInt(50));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+                System.out.println("tow sink finished!");
+                latch.countDown();
+            }
+        });
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 100; i++) {
+                    try {
+                        eventSink.sink(Arrays.asList(buildEntry("1", 1L + i, 30L + i)), address, "ljhtest3");
+                        Thread.sleep(50L + RandomUtils.nextInt(50));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+                System.out.println("tow sink finished!");
+                latch.countDown();
+            }
+        });
+
+        try {
+            latch.await();
+            Thread.sleep(200L);
+        } catch (InterruptedException e) {
+        }
+
+        eventSink.stop();
+        executor.shutdownNow();
+    }
+
+    private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        return entryBuilder.build();
+    }
+}
diff --git a/sink/src/test/java/com/alibaba/otter/canal/sink/stub/DummyEventStore.java b/sink/src/test/java/com/alibaba/otter/canal/sink/stub/DummyEventStore.java
new file mode 100644
index 00000000..8a52bb05
--- /dev/null
+++ b/sink/src/test/java/com/alibaba/otter/canal/sink/stub/DummyEventStore.java
@@ -0,0 +1,93 @@
+package com.alibaba.otter.canal.sink.stub;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+public class DummyEventStore implements CanalEventStore {
+
+    public void ack(Position position) throws CanalStoreException {
+
+    }
+
+    public Events get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
+        return null;
+    }
+
+    public Events get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
+                                                                                 CanalStoreException {
+        return null;
+    }
+
+    public Position getFirstPosition() throws CanalStoreException {
+        return null;
+    }
+
+    public Position getLatestPosition() throws CanalStoreException {
+        return null;
+    }
+
+    public void rollback() throws CanalStoreException {
+
+    }
+
+    public Events tryGet(Position start, int batchSize) throws CanalStoreException {
+        return null;
+    }
+
+    public boolean isStart() {
+        return false;
+    }
+
+    public void start() {
+
+    }
+
+    public void stop() {
+
+    }
+
+    public void cleanAll() throws CanalStoreException {
+    }
+
+    public void cleanUntil(Position position) throws CanalStoreException {
+
+    }
+
+    public void put(Event data) throws InterruptedException, CanalStoreException {
+        System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
+    }
+
+    public boolean put(Event data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
+        return true;
+    }
+
+    public boolean tryPut(Event data) throws CanalStoreException {
+        System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
+        return true;
+    }
+
+    public void put(List datas) throws InterruptedException, CanalStoreException {
+        Event data = datas.get(0);
+        System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
+    }
+
+    public boolean put(List datas, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        Event data = datas.get(0);
+        System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
+        return true;
+    }
+
+    public boolean tryPut(List datas) throws CanalStoreException {
+        Event data = datas.get(0);
+        System.out.println("time:" + data.getEntry().getHeader().getExecuteTime());
+        return true;
+    }
+
+}
diff --git a/store/pom.xml b/store/pom.xml
new file mode 100644
index 00000000..bb329d5f
--- /dev/null
+++ b/store/pom.xml
@@ -0,0 +1,36 @@
+
+	4.0.0
+	
+		com.alibaba.otter
+		canal
+		1.0.19-SNAPSHOT
+		../pom.xml
+	
+	com.alibaba.otter
+	canal.store
+	jar
+	canal store module for otter ${project.version}
+	
+		
+			com.alibaba.otter
+			canal.common
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.protocol
+			${project.version}
+		
+		
+			com.alibaba.otter
+			canal.meta
+			${project.version}
+		
+		
+		
+			junit
+			junit
+			test
+		
+	
+
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalGroupStore.java b/store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalGroupStore.java
new file mode 100644
index 00000000..57e63acd
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalGroupStore.java
@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.store;
+
+import java.util.Map;
+
+import org.springframework.util.Assert;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.google.common.collect.MapMaker;
+
+/**
+ * @author zebin.xuzb 2012-10-30 下午3:45:17
+ * @since 1.0.0
+ */
+public abstract class AbstractCanalGroupStore extends AbstractCanalLifeCycle implements CanalGroupEventStore {
+
+    protected Map stores = new MapMaker().makeMap();
+
+    @Override
+    public void addStoreInfo(StoreInfo info) {
+        checkInfo(info);
+        stores.put(info.getStoreName(), info);
+    }
+
+    protected void checkInfo(StoreInfo info) {
+        Assert.notNull(info);
+        Assert.hasText(info.getStoreName());
+    }
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalStoreScavenge.java b/store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalStoreScavenge.java
new file mode 100644
index 00000000..3145f25c
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/AbstractCanalStoreScavenge.java
@@ -0,0 +1,116 @@
+package com.alibaba.otter.canal.store;
+
+import java.util.List;
+
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.meta.CanalMetaManager;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+
+/**
+ * store回收机制
+ * 
+ * @author jianghang 2012-8-8 下午12:57:36
+ * @version 1.0.0
+ */
+public abstract class AbstractCanalStoreScavenge extends AbstractCanalLifeCycle implements CanalStoreScavenge {
+
+    protected String           destination;
+    protected CanalMetaManager canalMetaManager;
+    protected boolean          onAck            = true;
+    protected boolean          onFull           = false;
+    protected boolean          onSchedule       = false;
+    protected String           scavengeSchedule = null;
+
+    public void scavenge() {
+        Position position = getLatestAckPosition(destination);
+        cleanUntil(position);
+    }
+
+    /**
+     * 找出该destination中可被清理掉的position位置
+     * 
+     * @param destination
+     */
+    private Position getLatestAckPosition(String destination) {
+        List clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
+        LogPosition result = null;
+        if (!CollectionUtils.isEmpty(clientIdentitys)) {
+            // 尝试找到一个最小的logPosition
+            for (ClientIdentity clientIdentity : clientIdentitys) {
+                LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
+                if (position == null) {
+                    continue;
+                }
+
+                if (result == null) {
+                    result = position;
+                } else {
+                    result = min(result, position);
+                }
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * 找出一个最小的position位置
+     */
+    private LogPosition min(LogPosition position1, LogPosition position2) {
+        if (position1.getIdentity().equals(position2.getIdentity())) {
+            // 首先根据文件进行比较
+            if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
+                return position2;
+            } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
+                return position1;
+            } else {
+                // 根据offest进行比较
+                if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
+                    return position2;
+                } else {
+                    return position1;
+                }
+            }
+        } else {
+            // 不同的主备库，根据时间进行比较
+            if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
+                return position2;
+            } else {
+                return position1;
+            }
+        }
+    }
+
+    public void setOnAck(boolean onAck) {
+        this.onAck = onAck;
+    }
+
+    public void setOnFull(boolean onFull) {
+        this.onFull = onFull;
+    }
+
+    public void setOnSchedule(boolean onSchedule) {
+        this.onSchedule = onSchedule;
+    }
+
+    public String getScavengeSchedule() {
+        return scavengeSchedule;
+    }
+
+    public void setScavengeSchedule(String scavengeSchedule) {
+        this.scavengeSchedule = scavengeSchedule;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void setCanalMetaManager(CanalMetaManager canalMetaManager) {
+        this.canalMetaManager = canalMetaManager;
+    }
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/CanalEventStore.java b/store/src/main/java/com/alibaba/otter/canal/store/CanalEventStore.java
new file mode 100644
index 00000000..d83bcba0
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/CanalEventStore.java
@@ -0,0 +1,84 @@
+package com.alibaba.otter.canal.store;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.model.Events;
+
+/**
+ * canel数据存储接口
+ * 
+ * @author jianghang 2012-6-14 下午08:44:52
+ * @version 1.0.0
+ */
+public interface CanalEventStore extends CanalLifeCycle, CanalStoreScavenge {
+
+    /**
+     * 添加一组数据对象，阻塞等待其操作完成 (比如一次性添加一个事务数据)
+     */
+    void put(List data) throws InterruptedException, CanalStoreException;
+
+    /**
+     * 添加一组数据对象，阻塞等待其操作完成或者时间超时 (比如一次性添加一个事务数据)
+     */
+    boolean put(List data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException;
+
+    /**
+     * 添加一组数据对象 (比如一次性添加一个事务数据)
+     */
+    boolean tryPut(List data) throws CanalStoreException;
+
+    /**
+     * 添加一个数据对象，阻塞等待其操作完成
+     */
+    void put(T data) throws InterruptedException, CanalStoreException;
+
+    /**
+     * 添加一个数据对象，阻塞等待其操作完成或者时间超时
+     */
+    boolean put(T data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException;
+
+    /**
+     * 添加一个数据对象
+     */
+    boolean tryPut(T data) throws CanalStoreException;
+
+    /**
+     * 获取指定大小的数据，阻塞等待其操作完成
+     */
+    Events get(Position start, int batchSize) throws InterruptedException, CanalStoreException;
+
+    /**
+     * 获取指定大小的数据，阻塞等待其操作完成或者时间超时
+     */
+    Events get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
+                                                                             CanalStoreException;
+
+    /**
+     * 根据指定位置，获取一个指定大小的数据
+     */
+    Events tryGet(Position start, int batchSize) throws CanalStoreException;
+
+    /**
+     * 获取最后一条数据的position
+     */
+    Position getLatestPosition() throws CanalStoreException;
+
+    /**
+     * 获取第一条数据的position，如果没有数据返回为null
+     */
+    Position getFirstPosition() throws CanalStoreException;
+
+    /**
+     * 删除{@linkplain Position}之前的数据
+     */
+    void ack(Position position) throws CanalStoreException;
+
+    /**
+     * 出错时执行回滚操作(未提交ack的所有状态信息重新归位，减少出错时数据全部重来的成本)
+     */
+    void rollback() throws CanalStoreException;
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/CanalGroupEventStore.java b/store/src/main/java/com/alibaba/otter/canal/store/CanalGroupEventStore.java
new file mode 100644
index 00000000..17f57068
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/CanalGroupEventStore.java
@@ -0,0 +1,12 @@
+package com.alibaba.otter.canal.store;
+
+/**
+ * 提供给上层统一的 store 视图，内部则支持多种store混合，并且维持着多个store供上层进行路由
+ * 
+ * @author zebin.xuzb 2012-10-30 下午12:17:26
+ * @since 1.0.0
+ */
+public interface CanalGroupEventStore extends CanalEventStore {
+
+    void addStoreInfo(StoreInfo info);
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreConstants.java b/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreConstants.java
new file mode 100644
index 00000000..31cc88c8
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreConstants.java
@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.store;
+
+/**
+ * 常量值
+ * 
+ * @author jianghang 2012-6-14 下午09:40:33
+ * @version 1.0.0
+ */
+public interface CanalStoreConstants {
+
+    public static final String CODE_POSITION_NOT_FOUND    = "position:%s not found";
+
+    public static final String CODE_POSITION_NOT_IN_ORDER = "position:%s not in order";
+
+    public static final String ENCODING                   = "utf8";
+
+    public static final int    MAX_STORECOUNT             = 100;
+
+    public static final int    ROLLOVERCOUNT              = 100;
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreException.java b/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreException.java
new file mode 100644
index 00000000..c5618a41
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreException.java
@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.store;
+
+import com.alibaba.otter.canal.common.CanalException;
+
+/**
+ * canal 异常定义
+ * 
+ * @author jianghang 2012-6-15 下午04:57:35
+ * @version 1.0.0
+ */
+public class CanalStoreException extends CanalException {
+
+    private static final long serialVersionUID = -7288830284122672209L;
+
+    public CanalStoreException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalStoreException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalStoreException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalStoreException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalStoreException(Throwable cause){
+        super(cause);
+    }
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreScavenge.java b/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreScavenge.java
new file mode 100644
index 00000000..45260ad3
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/CanalStoreScavenge.java
@@ -0,0 +1,22 @@
+package com.alibaba.otter.canal.store;
+
+import com.alibaba.otter.canal.protocol.position.Position;
+
+/**
+ * store空间回收机制，信息采集以及控制何时调用{@linkplain CanalEventStore}.cleanUtil()接口
+ * 
+ * @author jianghang 2012-8-8 上午11:57:42
+ * @version 1.0.0
+ */
+public interface CanalStoreScavenge {
+
+    /**
+     * 清理position之前的数据
+     */
+    void cleanUntil(Position position) throws CanalStoreException;
+
+    /**
+     * 删除所有的数据
+     */
+    void cleanAll() throws CanalStoreException;
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/StoreInfo.java b/store/src/main/java/com/alibaba/otter/canal/store/StoreInfo.java
new file mode 100644
index 00000000..a3d1e01b
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/StoreInfo.java
@@ -0,0 +1,28 @@
+package com.alibaba.otter.canal.store;
+
+/**
+ * @author zebin.xuzb 2012-10-30 下午1:05:13
+ * @since 1.0.0
+ */
+public class StoreInfo {
+
+    private String storeName;
+    private String filter;
+
+    public String getStoreName() {
+        return storeName;
+    }
+
+    public String getFilter() {
+        return filter;
+    }
+
+    public void setStoreName(String storeName) {
+        this.storeName = storeName;
+    }
+
+    public void setFilter(String filter) {
+        this.filter = filter;
+    }
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/helper/CanalEventUtils.java b/store/src/main/java/com/alibaba/otter/canal/store/helper/CanalEventUtils.java
new file mode 100644
index 00000000..6e8bc8e9
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/helper/CanalEventUtils.java
@@ -0,0 +1,93 @@
+package com.alibaba.otter.canal.store.helper;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.position.EntryPosition;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.store.model.Event;
+
+/**
+ * 相关的操作工具
+ * 
+ * @author jianghang 2012-6-19 下午05:49:21
+ * @version 1.0.0
+ */
+public class CanalEventUtils {
+
+    /**
+     * 找出一个最小的position位置，相等的情况返回position1
+     */
+    public static LogPosition min(LogPosition position1, LogPosition position2) {
+        if (position1.getIdentity().equals(position2.getIdentity())) {
+            // 首先根据文件进行比较
+            if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
+                return position2;
+            } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
+                return position1;
+            } else {
+                // 根据offest进行比较
+                if (position1.getPostion().getPosition() > position2.getPostion().getPosition()) {
+                    return position2;
+                } else {
+                    return position1;
+                }
+            }
+        } else {
+            // 不同的主备库，根据时间进行比较
+            if (position1.getPostion().getTimestamp() > position2.getPostion().getTimestamp()) {
+                return position2;
+            } else {
+                return position1;
+            }
+        }
+    }
+
+    /**
+     * 根据entry创建对应的Position对象
+     */
+    public static LogPosition createPosition(Event event) {
+        EntryPosition position = new EntryPosition();
+        position.setJournalName(event.getEntry().getHeader().getLogfileName());
+        position.setPosition(event.getEntry().getHeader().getLogfileOffset());
+        position.setTimestamp(event.getEntry().getHeader().getExecuteTime());
+
+        LogPosition logPosition = new LogPosition();
+        logPosition.setPostion(position);
+        logPosition.setIdentity(event.getLogIdentity());
+        return logPosition;
+    }
+
+    /**
+     * 根据entry创建对应的Position对象
+     */
+    public static LogPosition createPosition(Event event, boolean included) {
+        EntryPosition position = new EntryPosition();
+        position.setJournalName(event.getEntry().getHeader().getLogfileName());
+        position.setPosition(event.getEntry().getHeader().getLogfileOffset());
+        position.setTimestamp(event.getEntry().getHeader().getExecuteTime());
+        position.setIncluded(included);
+
+        LogPosition logPosition = new LogPosition();
+        logPosition.setPostion(position);
+        logPosition.setIdentity(event.getLogIdentity());
+        return logPosition;
+    }
+
+    /**
+     * 判断当前的entry和position是否相同
+     */
+    public static boolean checkPosition(Event event, LogPosition logPosition) {
+        EntryPosition position = logPosition.getPostion();
+        CanalEntry.Entry entry = event.getEntry();
+        boolean result = position.getTimestamp().equals(entry.getHeader().getExecuteTime());
+
+        boolean exactely = (StringUtils.isBlank(position.getJournalName()) && position.getPosition() == null);
+        if (!exactely) {// 精确匹配
+            result &= StringUtils.equals(entry.getHeader().getLogfileName(), position.getJournalName());
+            result &= position.getPosition().equals(entry.getHeader().getLogfileOffset());
+        }
+
+        return result;
+    }
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java b/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java
new file mode 100644
index 00000000..99995a8f
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/memory/MemoryEventStoreWithBuffer.java
@@ -0,0 +1,566 @@
+package com.alibaba.otter.canal.store.memory;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.CanalEntry.EventType;
+import com.alibaba.otter.canal.protocol.position.LogPosition;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+import com.alibaba.otter.canal.store.AbstractCanalStoreScavenge;
+import com.alibaba.otter.canal.store.CanalEventStore;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.CanalStoreScavenge;
+import com.alibaba.otter.canal.store.helper.CanalEventUtils;
+import com.alibaba.otter.canal.store.model.BatchMode;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+/**
+ * 基于内存buffer构建内存memory store
+ * 
+ * 
+ * 变更记录：
+ * 1. 新增BatchMode类型，支持按内存大小获取批次数据，内存大小更加可控.
+ *   a. put操作，会首先根据bufferSize进行控制，然后再进行bufferSize * bufferMemUnit进行控制. 因存储的内容是以Event，如果纯依赖于memsize进行控制，会导致RingBuffer出现动态伸缩
+ * 

+ * 
+ * @author jianghang 2012-6-20 上午09:46:31
+ * @version 1.0.0
+ */
+public class MemoryEventStoreWithBuffer extends AbstractCanalStoreScavenge implements CanalEventStore, CanalStoreScavenge {
+
+    private static final long INIT_SQEUENCE = -1;
+    private int               bufferSize    = 16 * 1024;
+    private int               bufferMemUnit = 1024;                         // memsize的单位，默认为1kb大小
+    private int               indexMask;
+    private Event[]           entries;
+
+    // 记录下put/get/ack操作的三个下标
+    private AtomicLong        putSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前put操作最后一次写操作发生的位置
+    private AtomicLong        getSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前get操作读取的最后一条的位置
+    private AtomicLong        ackSequence   = new AtomicLong(INIT_SQEUENCE); // 代表当前ack操作的最后一条的位置
+
+    // 记录下put/get/ack操作的三个memsize大小
+    private AtomicLong        putMemSize    = new AtomicLong(0);
+    private AtomicLong        getMemSize    = new AtomicLong(0);
+    private AtomicLong        ackMemSize    = new AtomicLong(0);
+
+    // 阻塞put/get操作控制信号
+    private ReentrantLock     lock          = new ReentrantLock();
+    private Condition         notFull       = lock.newCondition();
+    private Condition         notEmpty      = lock.newCondition();
+
+    private BatchMode         batchMode     = BatchMode.ITEMSIZE;           // 默认为内存大小模式
+    private boolean           ddlIsolation  = false;
+
+    public MemoryEventStoreWithBuffer(){
+
+    }
+
+    public MemoryEventStoreWithBuffer(BatchMode batchMode){
+        this.batchMode = batchMode;
+    }
+
+    public void start() throws CanalStoreException {
+        super.start();
+        if (Integer.bitCount(bufferSize) != 1) {
+            throw new IllegalArgumentException("bufferSize must be a power of 2");
+        }
+
+        indexMask = bufferSize - 1;
+        entries = new Event[bufferSize];
+    }
+
+    public void stop() throws CanalStoreException {
+        super.stop();
+
+        cleanAll();
+    }
+
+    public void put(List data) throws InterruptedException, CanalStoreException {
+        if (data == null || data.isEmpty()) {
+            return;
+        }
+
+        final ReentrantLock lock = this.lock;
+        lock.lockInterruptibly();
+        try {
+            try {
+                while (!checkFreeSlotAt(putSequence.get() + data.size())) { // 检查是否有空位
+                    notFull.await(); // wait until not full
+                }
+            } catch (InterruptedException ie) {
+                notFull.signal(); // propagate to non-interrupted thread
+                throw ie;
+            }
+            doPut(data);
+            if (Thread.interrupted()) {
+                throw new InterruptedException();
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public boolean put(List data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        if (data == null || data.isEmpty()) {
+            return true;
+        }
+
+        long nanos = unit.toNanos(timeout);
+        final ReentrantLock lock = this.lock;
+        lock.lockInterruptibly();
+        try {
+            for (;;) {
+                if (checkFreeSlotAt(putSequence.get() + data.size())) {
+                    doPut(data);
+                    return true;
+                }
+                if (nanos <= 0) {
+                    return false;
+                }
+
+                try {
+                    nanos = notFull.awaitNanos(nanos);
+                } catch (InterruptedException ie) {
+                    notFull.signal(); // propagate to non-interrupted thread
+                    throw ie;
+                }
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public boolean tryPut(List data) throws CanalStoreException {
+        if (data == null || data.isEmpty()) {
+            return true;
+        }
+
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            if (!checkFreeSlotAt(putSequence.get() + data.size())) {
+                return false;
+            } else {
+                doPut(data);
+                return true;
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public void put(Event data) throws InterruptedException, CanalStoreException {
+        put(Arrays.asList(data));
+    }
+
+    public boolean put(Event data, long timeout, TimeUnit unit) throws InterruptedException, CanalStoreException {
+        return put(Arrays.asList(data), timeout, unit);
+    }
+
+    public boolean tryPut(Event data) throws CanalStoreException {
+        return tryPut(Arrays.asList(data));
+    }
+
+    /**
+     * 执行具体的put操作
+     */
+    private void doPut(List data) {
+        long current = putSequence.get();
+        long end = current + data.size();
+
+        // 先写数据，再更新对应的cursor,并发度高的情况，putSequence会被get请求可见，拿出了ringbuffer中的老的Entry值
+        for (long next = current + 1; next <= end; next++) {
+            entries[getIndex(next)] = data.get((int) (next - current - 1));
+        }
+
+        putSequence.set(end);
+
+        // 记录一下gets memsize信息，方便快速检索
+        if (batchMode.isMemSize()) {
+            long size = 0;
+            for (Event event : data) {
+                size += calculateSize(event);
+            }
+
+            putMemSize.getAndAdd(size);
+        }
+
+        // tell other threads that store is not empty
+        notEmpty.signal();
+    }
+
+    public Events get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lockInterruptibly();
+        try {
+            try {
+                while (!checkUnGetSlotAt((LogPosition) start, batchSize))
+                    notEmpty.await();
+            } catch (InterruptedException ie) {
+                notEmpty.signal(); // propagate to non-interrupted thread
+                throw ie;
+            }
+
+            return doGet(start, batchSize);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public Events get(Position start, int batchSize, long timeout, TimeUnit unit) throws InterruptedException,
+                                                                                        CanalStoreException {
+        long nanos = unit.toNanos(timeout);
+        final ReentrantLock lock = this.lock;
+        lock.lockInterruptibly();
+        try {
+            for (;;) {
+                if (checkUnGetSlotAt((LogPosition) start, batchSize)) {
+                    return doGet(start, batchSize);
+                }
+
+                if (nanos <= 0) {
+                    // 如果时间到了，有多少取多少
+                    return doGet(start, batchSize);
+                }
+
+                try {
+                    nanos = notEmpty.awaitNanos(nanos);
+                } catch (InterruptedException ie) {
+                    notEmpty.signal(); // propagate to non-interrupted thread
+                    throw ie;
+                }
+
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public Events tryGet(Position start, int batchSize) throws CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            return doGet(start, batchSize);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private Events doGet(Position start, int batchSize) throws CanalStoreException {
+        LogPosition startPosition = (LogPosition) start;
+
+        long current = getSequence.get();
+        long maxAbleSequence = putSequence.get();
+        long next = current;
+        long end = current;
+        // 如果startPosition为null，说明是第一次，默认+1处理
+        if (startPosition == null || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后，需要包含一下start位置，防止丢失第一条记录
+            next = next + 1;
+        }
+
+        if (current >= maxAbleSequence) {
+            return new Events();
+        }
+
+        Events result = new Events();
+        List entrys = result.getEvents();
+        long memsize = 0;
+        if (batchMode.isItemSize()) {
+            end = (next + batchSize - 1) < maxAbleSequence ? (next + batchSize - 1) : maxAbleSequence;
+            // 提取数据并返回
+            for (; next <= end; next++) {
+                Event event = entries[getIndex(next)];
+                if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
+                    // 如果是ddl隔离，直接返回
+                    if (entrys.size() == 0) {
+                        entrys.add(event);// 如果没有DML事件，加入当前的DDL事件
+                        end = next; // 更新end为当前
+                    } else {
+                        // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
+                        end = next - 1; // next-1一定大于current，不需要判断
+                    }
+                    break;
+                } else {
+                    entrys.add(event);
+                }
+            }
+        } else {
+            long maxMemSize = batchSize * bufferMemUnit;
+            for (; memsize <= maxMemSize && next <= maxAbleSequence; next++) {
+                // 永远保证可以取出第一条的记录，避免死锁
+                Event event = entries[getIndex(next)];
+                if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
+                    // 如果是ddl隔离，直接返回
+                    if (entrys.size() == 0) {
+                        entrys.add(event);// 如果没有DML事件，加入当前的DDL事件
+                        end = next; // 更新end为当前
+                    } else {
+                        // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
+                        end = next - 1; // next-1一定大于current，不需要判断
+                    }
+                    break;
+                } else {
+                    entrys.add(event);
+                    memsize += calculateSize(event);
+                    end = next;// 记录end位点
+                }
+            }
+
+        }
+
+        PositionRange range = new PositionRange();
+        result.setPositionRange(range);
+
+        range.setStart(CanalEventUtils.createPosition(entrys.get(0)));
+        range.setEnd(CanalEventUtils.createPosition(entrys.get(result.getEvents().size() - 1)));
+        // 记录一下是否存在可以被ack的点
+
+        for (int i = entrys.size() - 1; i >= 0; i--) {
+            Event event = entrys.get(i);
+            if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntry().getEntryType()
+                || CanalEntry.EntryType.TRANSACTIONEND == event.getEntry().getEntryType()
+                || isDdl(event.getEntry().getHeader().getEventType())) {
+                // 将事务头/尾设置可被为ack的点
+                range.setAck(CanalEventUtils.createPosition(event));
+                break;
+            }
+        }
+
+        if (getSequence.compareAndSet(current, end)) {
+            getMemSize.addAndGet(memsize);
+            notFull.signal();
+            return result;
+        } else {
+            return new Events();
+        }
+    }
+
+    public LogPosition getFirstPosition() throws CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            long firstSeqeuence = ackSequence.get();
+            if (firstSeqeuence == INIT_SQEUENCE && firstSeqeuence < putSequence.get()) {
+                // 没有ack过数据
+                Event event = entries[getIndex(firstSeqeuence + 1)]; // 最后一次ack为-1，需要移动到下一条,included
+                                                                     // = false
+                return CanalEventUtils.createPosition(event, false);
+            } else if (firstSeqeuence > INIT_SQEUENCE && firstSeqeuence < putSequence.get()) {
+                // ack未追上put操作
+                Event event = entries[getIndex(firstSeqeuence + 1)]; // 最后一次ack的位置数据
+                                                                     // + 1
+                return CanalEventUtils.createPosition(event, true);
+            } else if (firstSeqeuence > INIT_SQEUENCE && firstSeqeuence == putSequence.get()) {
+                // 已经追上，store中没有数据
+                Event event = entries[getIndex(firstSeqeuence)]; // 最后一次ack的位置数据，和last为同一条，included
+                                                                 // = false
+                return CanalEventUtils.createPosition(event, false);
+            } else {
+                // 没有任何数据
+                return null;
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public LogPosition getLatestPosition() throws CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            long latestSequence = putSequence.get();
+            if (latestSequence > INIT_SQEUENCE && latestSequence != ackSequence.get()) {
+                Event event = entries[(int) putSequence.get() & indexMask]; // 最后一次写入的数据，最后一条未消费的数据
+                return CanalEventUtils.createPosition(event, true);
+            } else if (latestSequence > INIT_SQEUENCE && latestSequence == ackSequence.get()) {
+                // ack已经追上了put操作
+                Event event = entries[(int) putSequence.get() & indexMask]; // 最后一次写入的数据，included
+                                                                            // =
+                                                                            // false
+                return CanalEventUtils.createPosition(event, false);
+            } else {
+                // 没有任何数据
+                return null;
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public void ack(Position position) throws CanalStoreException {
+        cleanUntil(position);
+    }
+
+    public void cleanUntil(Position position) throws CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            long sequence = ackSequence.get();
+            long maxSequence = getSequence.get();
+
+            boolean hasMatch = false;
+            long memsize = 0;
+            for (long next = sequence + 1; next <= maxSequence; next++) {
+                Event event = entries[getIndex(next)];
+                memsize += calculateSize(event);
+                boolean match = CanalEventUtils.checkPosition(event, (LogPosition) position);
+                if (match) {// 找到对应的position，更新ack seq
+                    hasMatch = true;
+
+                    if (batchMode.isMemSize()) {
+                        ackMemSize.addAndGet(memsize);
+                        // 尝试清空buffer中的内存，将ack之前的内存全部释放掉
+                        for (long index = sequence + 1; index < next; index++) {
+                            entries[getIndex(index)] = null;// 设置为null
+                        }
+                    }
+
+                    if (ackSequence.compareAndSet(sequence, next)) {// 避免并发ack
+                        notFull.signal();
+                        return;
+                    }
+                }
+            }
+
+            if (!hasMatch) {// 找不到对应需要ack的position
+                throw new CanalStoreException("no match ack position" + position.toString());
+            }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public void rollback() throws CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            getSequence.set(ackSequence.get());
+            getMemSize.set(ackMemSize.get());
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    public void cleanAll() throws CanalStoreException {
+        final ReentrantLock lock = this.lock;
+        lock.lock();
+        try {
+            putSequence.set(INIT_SQEUENCE);
+            getSequence.set(INIT_SQEUENCE);
+            ackSequence.set(INIT_SQEUENCE);
+
+            putMemSize.set(0);
+            getMemSize.set(0);
+            ackMemSize.set(0);
+            entries = null;
+            // for (int i = 0; i < entries.length; i++) {
+            // entries[i] = null;
+            // }
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    // =================== helper method =================
+
+    private long getMinimumGetOrAck() {
+        long get = getSequence.get();
+        long ack = ackSequence.get();
+        return ack <= get ? ack : get;
+    }
+
+    /**
+     * 查询是否有空位
+     */
+    private boolean checkFreeSlotAt(final long sequence) {
+        final long wrapPoint = sequence - bufferSize;
+        final long minPoint = getMinimumGetOrAck();
+        if (wrapPoint > minPoint) { // 刚好追上一轮
+            return false;
+        } else {
+            // 在bufferSize模式上，再增加memSize控制
+            if (batchMode.isMemSize()) {
+                final long memsize = putMemSize.get() - ackMemSize.get();
+                if (memsize < bufferSize * bufferMemUnit) {
+                    return true;
+                } else {
+                    return false;
+                }
+            } else {
+                return true;
+            }
+        }
+    }
+
+    /**
+     * 检查是否存在需要get的数据,并且数量>=batchSize
+     */
+    private boolean checkUnGetSlotAt(LogPosition startPosition, int batchSize) {
+        if (batchMode.isItemSize()) {
+            long current = getSequence.get();
+            long maxAbleSequence = putSequence.get();
+            long next = current;
+            if (startPosition == null || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后，需要包含一下start位置，防止丢失第一条记录
+                next = next + 1;// 少一条数据
+            }
+
+            if (current < maxAbleSequence && next + batchSize - 1 <= maxAbleSequence) {
+                return true;
+            } else {
+                return false;
+            }
+        } else {
+            // 处理内存大小判断
+            long currentSize = getMemSize.get();
+            long maxAbleSize = putMemSize.get();
+
+            if (maxAbleSize - currentSize >= batchSize * bufferMemUnit) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+    }
+
+    private long calculateSize(Event event) {
+        // 直接返回binlog中的事件大小
+        return event.getEntry().getHeader().getEventLength();
+    }
+
+    private int getIndex(long sequcnce) {
+        return (int) sequcnce & indexMask;
+    }
+
+    private boolean isDdl(EventType type) {
+        return type == EventType.ALTER || type == EventType.CREATE || type == EventType.ERASE
+               || type == EventType.RENAME || type == EventType.TRUNCATE || type == EventType.CINDEX
+               || type == EventType.DINDEX;
+    }
+
+    // ================ setter / getter ==================
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
+
+    public void setBufferMemUnit(int bufferMemUnit) {
+        this.bufferMemUnit = bufferMemUnit;
+    }
+
+    public void setBatchMode(BatchMode batchMode) {
+        this.batchMode = batchMode;
+    }
+
+    public void setDdlIsolation(boolean ddlIsolation) {
+        this.ddlIsolation = ddlIsolation;
+    }
+
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/model/BatchMode.java b/store/src/main/java/com/alibaba/otter/canal/store/model/BatchMode.java
new file mode 100644
index 00000000..8036a89f
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/model/BatchMode.java
@@ -0,0 +1,24 @@
+package com.alibaba.otter.canal.store.model;
+
+/**
+ * 批处理模式
+ * 
+ * @author jianghang 2013-3-18 上午11:51:15
+ * @version 1.0.3
+ */
+public enum BatchMode {
+
+    /** 对象数量 */
+    ITEMSIZE,
+
+    /** 内存大小 */
+    MEMSIZE;
+
+    public boolean isItemSize() {
+        return this == BatchMode.ITEMSIZE;
+    }
+
+    public boolean isMemSize() {
+        return this == BatchMode.MEMSIZE;
+    }
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java b/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java
new file mode 100644
index 00000000..c31d37f3
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/model/Event.java
@@ -0,0 +1,50 @@
+package com.alibaba.otter.canal.store.model;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+
+/**
+ * store存储数据对象
+ * 
+ * @author jianghang 2012-7-13 下午03:03:03
+ */
+public class Event implements Serializable {
+
+    private static final long serialVersionUID = 1333330351758762739L;
+
+    private LogIdentity       logIdentity;                            // 记录数据产生的来源
+    private CanalEntry.Entry  entry;
+
+    public Event(){
+    }
+
+    public Event(LogIdentity logIdentity, CanalEntry.Entry entry){
+        this.logIdentity = logIdentity;
+        this.entry = entry;
+    }
+
+    public LogIdentity getLogIdentity() {
+        return logIdentity;
+    }
+
+    public void setLogIdentity(LogIdentity logIdentity) {
+        this.logIdentity = logIdentity;
+    }
+
+    public CanalEntry.Entry getEntry() {
+        return entry;
+    }
+
+    public void setEntry(CanalEntry.Entry entry) {
+        this.entry = entry;
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+}
diff --git a/store/src/main/java/com/alibaba/otter/canal/store/model/Events.java b/store/src/main/java/com/alibaba/otter/canal/store/model/Events.java
new file mode 100644
index 00000000..aa3b6af3
--- /dev/null
+++ b/store/src/main/java/com/alibaba/otter/canal/store/model/Events.java
@@ -0,0 +1,44 @@
+package com.alibaba.otter.canal.store.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+import com.alibaba.otter.canal.protocol.position.PositionRange;
+
+/**
+ * 代表一组数据对象的集合
+ * 
+ * @author jianghang 2012-6-14 下午09:07:41
+ * @version 1.0.0
+ */
+public class Events implements Serializable {
+
+    private static final long serialVersionUID = -7337454954300706044L;
+
+    private PositionRange     positionRange    = new PositionRange();
+    private List       events           = new ArrayList();
+
+    public List getEvents() {
+        return events;
+    }
+
+    public void setEvents(List events) {
+        this.events = events;
+    }
+
+    public PositionRange getPositionRange() {
+        return positionRange;
+    }
+
+    public void setPositionRange(PositionRange positionRange) {
+        this.positionRange = positionRange;
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+}
diff --git a/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreBase.java b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreBase.java
new file mode 100644
index 00000000..a0a27ca3
--- /dev/null
+++ b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreBase.java
@@ -0,0 +1,49 @@
+package com.alibaba.otter.cancel.store.memory.buffer;
+
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalEntry.Header;
+import com.alibaba.otter.canal.protocol.position.LogIdentity;
+import com.alibaba.otter.canal.store.model.Event;
+
+public class MemoryEventStoreBase {
+
+    private static final String MYSQL_ADDRESS = "127.0.0.1";
+
+    protected void sleep(Long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail();
+        }
+    }
+
+    protected Event buildEvent(String binlogFile, long offset, long timestamp) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        headerBuilder.setEventLength(1024);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        Entry entry = entryBuilder.build();
+
+        return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
+    }
+
+    protected Event buildEvent(String binlogFile, long offset, long timestamp, long eventLenght) {
+        Header.Builder headerBuilder = Header.newBuilder();
+        headerBuilder.setLogfileName(binlogFile);
+        headerBuilder.setLogfileOffset(offset);
+        headerBuilder.setExecuteTime(timestamp);
+        headerBuilder.setEventLength(eventLenght);
+        Entry.Builder entryBuilder = Entry.newBuilder();
+        entryBuilder.setHeader(headerBuilder.build());
+        Entry entry = entryBuilder.build();
+
+        return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
+    }
+}
diff --git a/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMemBatchTest.java b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMemBatchTest.java
new file mode 100644
index 00000000..eb5aa1fa
--- /dev/null
+++ b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMemBatchTest.java
@@ -0,0 +1,317 @@
+package com.alibaba.otter.cancel.store.memory.buffer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.helper.CanalEventUtils;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import com.alibaba.otter.canal.store.model.BatchMode;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+public class MemoryEventStoreMemBatchTest extends MemoryEventStoreBase {
+
+    @Test
+    public void testOnePut() {
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+        // 尝试阻塞
+        try {
+            eventStore.put(buildEvent("1", 1L, 1L, 1024));
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        }
+        // 尝试阻塞+超时
+        boolean result = false;
+        try {
+            result = eventStore.put(buildEvent("1", 1L, 1L), 1000L, TimeUnit.MILLISECONDS);
+            Assert.assertTrue(result);
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        }
+        // 尝试
+        result = eventStore.tryPut(buildEvent("1", 1L, 1L));
+        Assert.assertTrue(result);
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testOnePutExceedLimit() {
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(1);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+        // 尝试阻塞
+        try {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L, 1025));// 只有一条记录，第一条超过也允许放入
+            Assert.assertTrue(result);
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        }
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testFullPut() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            Assert.assertTrue(result);
+        }
+
+        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + bufferSize));
+        Assert.assertFalse(result);
+
+        try {
+            result = eventStore.put(buildEvent("1", 1L, 1L + bufferSize), 1000L, TimeUnit.MILLISECONDS);
+        } catch (CanalStoreException e) {
+            Assert.fail(e.getMessage());
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        Assert.assertFalse(result);
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testOnePutOneGet() {
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+
+        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
+        Assert.assertTrue(result);
+
+        Position position = eventStore.getFirstPosition();
+        Events entrys = eventStore.tryGet(position, 1);
+        Assert.assertTrue(entrys.getEvents().size() == 1);
+        Assert.assertEquals(position, entrys.getPositionRange().getStart());
+        Assert.assertEquals(position, entrys.getPositionRange().getEnd());
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testFullPutBatchGet() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        Position first = eventStore.getFirstPosition();
+        Position lastest = eventStore.getLatestPosition();
+        Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));
+
+        System.out.println("start get");
+        Events entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("first get size : " + entrys1.getEvents().size());
+
+        Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
+        Assert.assertEquals(first, entrys1.getPositionRange().getStart());
+        Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());
+
+        Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
+        eventStore.stop();
+    }
+
+    @Test
+    public void testBlockPutOneGet() {
+        final MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(16);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+
+        final int batchSize = 10;
+        for (int i = 0; i < batchSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
+            Assert.assertTrue(result);
+        }
+
+        final Position position = eventStore.getFirstPosition();
+        try {
+            Events entrys = eventStore.get(position, batchSize);
+            Assert.assertTrue(entrys.getEvents().size() == batchSize);
+            Assert.assertEquals(position, entrys.getPositionRange().getStart());
+            Assert.assertEquals(position, entrys.getPositionRange().getEnd());
+        } catch (CanalStoreException e) {
+        } catch (InterruptedException e) {
+        }
+
+        ExecutorService executor = Executors.newFixedThreadPool(1);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                boolean result = false;
+                try {
+                    eventStore.get(position, batchSize);
+                } catch (CanalStoreException e) {
+                } catch (InterruptedException e) {
+                    System.out.println("interrupt occured.");
+                    result = true;
+                }
+                Assert.assertTrue(result);
+            }
+        });
+
+        try {
+            Thread.sleep(1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        executor.shutdownNow();
+
+        try {
+            Thread.sleep(1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        eventStore.stop();
+    }
+
+    @Test
+    public void testRollback() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize / 2; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        sleep(50L);
+        Position first = eventStore.getFirstPosition();
+        Position lastest = eventStore.getLatestPosition();
+        Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
+
+        System.out.println("start get");
+        Events entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("first get size : " + entrys1.getEvents().size());
+
+        eventStore.rollback();
+
+        entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("after rollback get size : " + entrys1.getEvents().size());
+        Assert.assertTrue(entrys1.getEvents().size() == bufferSize / 2);
+
+        // 继续造数据
+        for (int i = bufferSize / 2; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        Events entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("second get size : " + entrys2.getEvents().size());
+
+        eventStore.rollback();
+
+        entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("after rollback get size : " + entrys2.getEvents().size());
+        Assert.assertTrue(entrys2.getEvents().size() == bufferSize);
+
+        first = eventStore.getFirstPosition();
+        lastest = eventStore.getLatestPosition();
+        List entrys = new ArrayList(entrys2.getEvents());
+        Assert.assertTrue(entrys.size() == bufferSize);
+        Assert.assertEquals(first, entrys2.getPositionRange().getStart());
+        Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
+
+        Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(bufferSize - 1)));
+        eventStore.stop();
+    }
+
+    @Test
+    public void testAck() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize / 2; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        sleep(50L);
+        Position first = eventStore.getFirstPosition();
+        Position lastest = eventStore.getLatestPosition();
+        Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
+
+        System.out.println("start get");
+        Events entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("first get size : " + entrys1.getEvents().size());
+
+        eventStore.cleanUntil(entrys1.getPositionRange().getEnd());
+        sleep(50L);
+
+        // 继续造数据
+        for (int i = bufferSize / 2; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        Events entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("second get size : " + entrys2.getEvents().size());
+
+        eventStore.rollback();
+
+        entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("after rollback get size : " + entrys2.getEvents().size());
+
+        first = eventStore.getFirstPosition();
+        lastest = eventStore.getLatestPosition();
+        List entrys = new ArrayList(entrys2.getEvents());
+        Assert.assertEquals(first, entrys2.getPositionRange().getStart());
+        Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
+
+        Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(entrys.size() - 1)));
+
+        // 全部ack掉
+        eventStore.cleanUntil(entrys2.getPositionRange().getEnd());
+
+        // 最后就拿不到数据
+        Events entrys3 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("third get size : " + entrys3.getEvents().size());
+        Assert.assertEquals(0, entrys3.getEvents().size());
+
+        eventStore.stop();
+    }
+}
diff --git a/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java
new file mode 100644
index 00000000..754f7218
--- /dev/null
+++ b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreMultiThreadTest.java
@@ -0,0 +1,191 @@
+package com.alibaba.otter.cancel.store.memory.buffer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.Assert;
+
+import org.apache.commons.lang.math.RandomUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.util.CollectionUtils;
+
+import com.alibaba.otter.canal.common.utils.BooleanMutex;
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import com.alibaba.otter.canal.store.model.BatchMode;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+/**
+ * 多线程的put/get/ack/rollback测试
+ * 
+ * @author jianghang 2012-6-20 下午02:50:36
+ * @version 1.0.0
+ */
+public class MemoryEventStoreMultiThreadTest extends MemoryEventStoreBase {
+
+    private ExecutorService            executor = Executors.newFixedThreadPool(2); // 1 producer ,1 cousmer
+    private MemoryEventStoreWithBuffer eventStore;
+
+    @Before
+    public void setUp() {
+        eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(16 * 16);
+        eventStore.setBatchMode(BatchMode.MEMSIZE);
+        eventStore.start();
+    }
+
+    @After
+    public void tearDown() {
+        eventStore.stop();
+    }
+
+    @Test
+    public void test() {
+        CountDownLatch latch = new CountDownLatch(1);
+        BooleanMutex mutex = new BooleanMutex(true);
+        Producer producer = new Producer(mutex, 10);
+        Cosumer cosumer = new Cosumer(latch, 20, 50);
+
+        executor.submit(producer);
+        executor.submit(cosumer);
+
+        try {
+            Thread.sleep(30 * 1000L);
+        } catch (InterruptedException e) {
+        }
+
+        mutex.set(false);
+        try {
+            latch.await();
+        } catch (InterruptedException e) {
+        }
+        executor.shutdown();
+
+        List result = cosumer.getResult();
+
+        Long last = -1L;
+        for (Long offest : result) {
+            Assert.assertTrue(last + 1 == offest);// 取出来的数据一定是递增的
+            last = offest;
+        }
+    }
+
+    class Producer implements Runnable {
+
+        private BooleanMutex mutex;
+        private int          freq;
+
+        public Producer(BooleanMutex mutex, int freq){
+            this.mutex = mutex;
+            this.freq = freq;
+        }
+
+        public void run() {
+            long offest = 0;
+            while (true) {
+                try {
+                    mutex.get();
+                    Thread.sleep(RandomUtils.nextInt(freq));
+                } catch (InterruptedException e) {
+                    return;
+                }
+                Event event = buildEvent("1", offest++, 1L);
+
+                try {
+                    Thread.sleep(RandomUtils.nextInt(freq));
+                } catch (InterruptedException e) {
+                    return;
+                }
+                try {
+                    eventStore.put(event);
+                } catch (CanalStoreException e) {
+                } catch (InterruptedException e) {
+                }
+            }
+        }
+    }
+
+    class Cosumer implements Runnable {
+
+        private CountDownLatch latch;
+        private int            freq;
+        private int            batchSize;
+        private List     result = new ArrayList();
+
+        public Cosumer(CountDownLatch latch, int freq, int batchSize){
+            this.latch = latch;
+            this.freq = freq;
+            this.batchSize = batchSize;
+        }
+
+        public void run() {
+            Position first = eventStore.getFirstPosition();
+            while (first == null) {
+                try {
+                    Thread.sleep(RandomUtils.nextInt(freq));
+                } catch (InterruptedException e) {
+                    latch.countDown();
+                    return;
+                }
+
+                first = eventStore.getFirstPosition();
+            }
+
+            int ackCount = 0;
+            int emptyCount = 0;
+            while (emptyCount < 10) {
+                try {
+                    Thread.sleep(RandomUtils.nextInt(freq));
+                } catch (InterruptedException e) {
+                }
+
+                try {
+                    Events entrys = eventStore.get(first, batchSize, 1000L, TimeUnit.MILLISECONDS);
+                    // Events entrys = eventStore.tryGet(first, batchSize);
+                    if (!CollectionUtils.isEmpty(entrys.getEvents())) {
+                        if (entrys.getEvents().size() != batchSize) {
+                            System.out.println("get size:" + entrys.getEvents().size() + " with not full batchSize:"
+                                               + batchSize);
+                        }
+
+                        first = entrys.getPositionRange().getEnd();
+                        for (Event event : entrys.getEvents()) {
+                            this.result.add(event.getEntry().getHeader().getLogfileOffset());
+                        }
+                        emptyCount = 0;
+
+                        System.out.println("offest : "
+                                           + entrys.getEvents().get(0).getEntry().getHeader().getLogfileOffset()
+                                           + " , count :" + entrys.getEvents().size());
+                        ackCount++;
+                        if (ackCount == 1) {
+                            eventStore.cleanUntil(entrys.getPositionRange().getEnd());
+                            System.out.println("first position : " + eventStore.getFirstPosition());
+                            ackCount = 0;
+                        }
+                    } else {
+                        emptyCount++;
+                        System.out.println("empty events for " + emptyCount);
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+
+            latch.countDown();
+        }
+
+        public List getResult() {
+            return result;
+        }
+
+    }
+}
diff --git a/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStorePutAndGetTest.java b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStorePutAndGetTest.java
new file mode 100644
index 00000000..b030a1e1
--- /dev/null
+++ b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStorePutAndGetTest.java
@@ -0,0 +1,177 @@
+package com.alibaba.otter.cancel.store.memory.buffer;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.CanalStoreException;
+import com.alibaba.otter.canal.store.helper.CanalEventUtils;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+/**
+ * 测试普通的put / get操作
+ * 
+ * @author jianghang 2012-6-19 下午09:50:08
+ * @version 1.0.0
+ */
+public class MemoryEventStorePutAndGetTest extends MemoryEventStoreBase {
+
+    @Test
+    public void testOnePut() {
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.start();
+        // 尝试阻塞
+        try {
+            eventStore.put(buildEvent("1", 1L, 1L));
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        }
+        // 尝试阻塞+超时
+        boolean result = false;
+        try {
+            result = eventStore.put(buildEvent("1", 1L, 1L), 1000L, TimeUnit.MILLISECONDS);
+            Assert.assertTrue(result);
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        }
+        // 尝试
+        result = eventStore.tryPut(buildEvent("1", 1L, 1L));
+        Assert.assertTrue(result);
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testFullPut() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            Assert.assertTrue(result);
+        }
+
+        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + bufferSize));
+        Assert.assertFalse(result);
+
+        try {
+            result = eventStore.put(buildEvent("1", 1L, 1L + bufferSize), 1000L, TimeUnit.MILLISECONDS);
+        } catch (CanalStoreException e) {
+            Assert.fail(e.getMessage());
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+
+        Assert.assertFalse(result);
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testOnePutOneGet() {
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.start();
+
+        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
+        Assert.assertTrue(result);
+
+        Position position = eventStore.getFirstPosition();
+        Events entrys = eventStore.tryGet(position, 1);
+        Assert.assertTrue(entrys.getEvents().size() == 1);
+        Assert.assertEquals(position, entrys.getPositionRange().getStart());
+        Assert.assertEquals(position, entrys.getPositionRange().getEnd());
+
+        eventStore.stop();
+    }
+
+    @Test
+    public void testFullPutBatchGet() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        Position first = eventStore.getFirstPosition();
+        Position lastest = eventStore.getLatestPosition();
+        Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));
+
+        System.out.println("start get");
+        Events entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("first get size : " + entrys1.getEvents().size());
+
+        Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
+        Assert.assertEquals(first, entrys1.getPositionRange().getStart());
+        Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());
+
+        Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
+        eventStore.stop();
+    }
+
+    @Test
+    public void testBlockPutOneGet() {
+        final MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.start();
+
+        final int batchSize = 10;
+        for (int i = 0; i < batchSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
+            Assert.assertTrue(result);
+        }
+
+        final Position position = eventStore.getFirstPosition();
+        try {
+            Events entrys = eventStore.get(position, batchSize);
+            Assert.assertTrue(entrys.getEvents().size() == batchSize);
+            Assert.assertEquals(position, entrys.getPositionRange().getStart());
+            Assert.assertEquals(position, entrys.getPositionRange().getEnd());
+        } catch (CanalStoreException e) {
+        } catch (InterruptedException e) {
+        }
+
+        ExecutorService executor = Executors.newFixedThreadPool(1);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                boolean result = false;
+                try {
+                    eventStore.get(position, batchSize);
+                } catch (CanalStoreException e) {
+                } catch (InterruptedException e) {
+                    System.out.println("interrupt occured.");
+                    result = true;
+                }
+                Assert.assertTrue(result);
+            }
+        });
+
+        try {
+            Thread.sleep(1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        executor.shutdownNow();
+
+        try {
+            Thread.sleep(1000L);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+        eventStore.stop();
+    }
+}
diff --git a/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreRollbackAndAckTest.java b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreRollbackAndAckTest.java
new file mode 100644
index 00000000..29270f8c
--- /dev/null
+++ b/store/src/test/java/com/alibaba/otter/cancel/store/memory/buffer/MemoryEventStoreRollbackAndAckTest.java
@@ -0,0 +1,141 @@
+package com.alibaba.otter.cancel.store.memory.buffer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.protocol.position.Position;
+import com.alibaba.otter.canal.store.helper.CanalEventUtils;
+import com.alibaba.otter.canal.store.memory.MemoryEventStoreWithBuffer;
+import com.alibaba.otter.canal.store.model.Event;
+import com.alibaba.otter.canal.store.model.Events;
+
+/**
+ * 测试下rollback / ack的操作
+ * 
+ * @author jianghang 2012-6-19 下午09:49:28
+ * @version 1.0.0
+ */
+public class MemoryEventStoreRollbackAndAckTest extends MemoryEventStoreBase {
+
+    @Test
+    public void testRollback() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize / 2; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        sleep(50L);
+        Position first = eventStore.getFirstPosition();
+        Position lastest = eventStore.getLatestPosition();
+        Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
+
+        System.out.println("start get");
+        Events entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("first get size : " + entrys1.getEvents().size());
+
+        eventStore.rollback();
+
+        entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("after rollback get size : " + entrys1.getEvents().size());
+        Assert.assertTrue(entrys1.getEvents().size() == bufferSize / 2);
+
+        // 继续造数据
+        for (int i = bufferSize / 2; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        Events entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("second get size : " + entrys2.getEvents().size());
+
+        eventStore.rollback();
+
+        entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("after rollback get size : " + entrys2.getEvents().size());
+        Assert.assertTrue(entrys2.getEvents().size() == bufferSize);
+
+        first = eventStore.getFirstPosition();
+        lastest = eventStore.getLatestPosition();
+        List entrys = new ArrayList(entrys2.getEvents());
+        Assert.assertTrue(entrys.size() == bufferSize);
+        Assert.assertEquals(first, entrys2.getPositionRange().getStart());
+        Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
+
+        Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(bufferSize - 1)));
+        eventStore.stop();
+    }
+
+    @Test
+    public void testAck() {
+        int bufferSize = 16;
+        MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
+        eventStore.setBufferSize(bufferSize);
+        eventStore.start();
+
+        for (int i = 0; i < bufferSize / 2; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        sleep(50L);
+        Position first = eventStore.getFirstPosition();
+        Position lastest = eventStore.getLatestPosition();
+        Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
+
+        System.out.println("start get");
+        Events entrys1 = eventStore.tryGet(first, bufferSize);
+        System.out.println("first get size : " + entrys1.getEvents().size());
+
+        eventStore.cleanUntil(entrys1.getPositionRange().getEnd());
+        sleep(50L);
+
+        // 继续造数据
+        for (int i = bufferSize / 2; i < bufferSize; i++) {
+            boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
+            sleep(100L);
+            Assert.assertTrue(result);
+        }
+
+        Events entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("second get size : " + entrys2.getEvents().size());
+
+        eventStore.rollback();
+
+        entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("after rollback get size : " + entrys2.getEvents().size());
+
+        first = eventStore.getFirstPosition();
+        lastest = eventStore.getLatestPosition();
+        List entrys = new ArrayList(entrys2.getEvents());
+        Assert.assertEquals(first, entrys2.getPositionRange().getStart());
+        Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
+
+        Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
+        Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(entrys.size() - 1)));
+
+        // 全部ack掉
+        eventStore.cleanUntil(entrys2.getPositionRange().getEnd());
+
+        // 最后就拿不到数据
+        Events entrys3 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
+        System.out.println("third get size : " + entrys3.getEvents().size());
+        Assert.assertEquals(0, entrys3.getEvents().size());
+
+        eventStore.stop();
+    }
+
+}