init
This commit is contained in:
agapple
+15
@@ -0,0 +1,15 @@
|
|||||||
|
.svn/
|
||||||
|
target/
|
||||||
|
test-output/
|
||||||
|
*.class
|
||||||
|
.classpath
|
||||||
|
.project
|
||||||
|
.settings/
|
||||||
|
tmp
|
||||||
|
temp
|
||||||
|
*.log
|
||||||
|
antx.properties
|
||||||
|
otter.properties
|
||||||
|
jtester.properties
|
||||||
|
.idea/
|
||||||
|
*.iml
|
||||||
+202
@@ -0,0 +1,202 @@
|
|||||||
|
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
<div class="blog_content">
|
||||||
|
<div class="iteye-blog-content-contain">
|
||||||
|
|
||||||
|
<h3>最新更新</h3>
|
||||||
|
<ol>
|
||||||
|
<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
|
||||||
|
<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
|
||||||
|
</ol>
|
||||||
|
|
||||||
|
<h1>背景</h1>
|
||||||
|
<p style="font-size: 14px;"> 早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&消费的业务，从此开启了一段新纪元。</p>
|
||||||
|
<p style="font-size: 14px;"> ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.6及以下的版本(阿里内部mysql 5.6.10, mysql 5.5.18和5.1.40/48)</p>
|
||||||
|
<p style="font-size: 14px;"> </p>
|
||||||
|
<p style="font-size: 14px;">基于日志增量订阅&消费支持的业务：</p>
|
||||||
|
<ol style="font-size: 14px;">
|
||||||
|
<li>数据库镜像</li>
|
||||||
|
<li>数据库实时备份</li>
|
||||||
|
<li>多级索引 (卖家和买家各自分库索引)</li>
|
||||||
|
<li>search build</li>
|
||||||
|
<li>业务cache刷新</li>
|
||||||
|
<li>价格变化等重要业务消息</li>
|
||||||
|
</ol>
|
||||||
|
<h1>项目介绍</h1>
|
||||||
|
<p style="font-size: 14px;"> 名称：canal [kə'næl]</p>
|
||||||
|
<p style="font-size: 14px;"> 译意： 水道/管道/沟渠 </p>
|
||||||
|
<p style="font-size: 14px;"> 语言： 纯java开发</p>
|
||||||
|
<p style="font-size: 14px;"> 定位： 基于数据库增量日志解析，提供增量数据订阅&消费，目前主要支持了mysql</p>
|
||||||
|
<p style="font-size: 14px;"> 关键词： mysql binlog parser / real-time / queue&topic </p>
|
||||||
|
<p style="font-size: 14px;"> </p>
|
||||||
|
<h2>工作原理</h2>
|
||||||
|
<h3 style="font-size: 14px;">mysql主备复制实现</h3>
|
||||||
|
<p><img src="http://dl.iteye.com/upload/attachment/0080/3086/468c1a14-e7ad-3290-9d3d-44ac501a7227.jpg" alt=""><br> 从上层来看，复制分成三步：
|
||||||
|
<ol>
|
||||||
|
<li>master将改变记录到二进制日志(binary log)中（这些记录叫做二进制日志事件，binary log events，可以通过show binlog events进行查看）；</li>
|
||||||
|
<li>slave将master的binary log events拷贝到它的中继日志(relay log)；</li>
|
||||||
|
<li>slave重做中继日志中的事件，将改变反映它自己的数据。</li>
|
||||||
|
</ol>
|
||||||
|
<h3>canal的工作原理：</h3>
|
||||||
|
<p><img width="590" src="http://dl.iteye.com/upload/attachment/0080/3107/c87b67ba-394c-3086-9577-9db05be04c95.jpg" alt="" height="273">
|
||||||
|
<p>原理相对比较简单：</p>
|
||||||
|
<ol>
|
||||||
|
<li>canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议</li>
|
||||||
|
<li>mysql master收到dump请求，开始推送binary log给slave(也就是canal)</li>
|
||||||
|
<li>canal解析binary log对象(原始为byte流)</li>
|
||||||
|
</ol>
|
||||||
|
|
||||||
|
<h1>相关文档</h1>
|
||||||
|
|
||||||
|
See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文档</a>
|
||||||
|
|
||||||
|
<h3><a name="table-of-contents" class="anchor" href="#table-of-contents"><span class="mini-icon mini-icon-link"></span></a>wiki文档列表</h3>
|
||||||
|
<ul>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Home">Home</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Introduction">Introduction</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/QuickStart">QuickStart</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientExample">ClientExample</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/AdminGuide">AdminGuide</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientAPI">ClientAPI</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/DevGuide">DevGuide</a></li>
|
||||||
|
<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/BinlogChange%28mysql5.6%29">BinlogChange(Mysql5.6)</a></li>
|
||||||
|
<li><a href="http://alibaba.github.com/canal/release.html">ReleaseNotes</a></li>
|
||||||
|
<li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h1>问题反馈</h1>
|
||||||
|
<ol>
|
||||||
|
<li>qq交流群： 161559791 </li>
|
||||||
|
<li>邮件交流： jianghang115@gmail.com </li>
|
||||||
|
<li>新浪微博： agapple0002 </li>
|
||||||
|
<li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
|
||||||
|
</ol>
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
release link : https://alibaba.github.com/canal/release.html
|
||||||
|
|
||||||
|
download link : https://github.com/alibaba/canal/releases
|
||||||
+101
@@ -0,0 +1,101 @@
|
|||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal</artifactId>
|
||||||
|
<version>1.0.19-SNAPSHOT</version>
|
||||||
|
<relativePath>../pom.xml</relativePath>
|
||||||
|
</parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal.client</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
<name>canal client module for otter ${project.version}</name>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal.protocol</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- junit -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>dev</id>
|
||||||
|
<activation>
|
||||||
|
<activeByDefault>true</activeByDefault>
|
||||||
|
<property>
|
||||||
|
<name>env</name>
|
||||||
|
<value>!javadoc</value>
|
||||||
|
</property>
|
||||||
|
</activation>
|
||||||
|
</profile>
|
||||||
|
|
||||||
|
<profile>
|
||||||
|
<id>javadoc</id>
|
||||||
|
<activation>
|
||||||
|
<property>
|
||||||
|
<name>env</name>
|
||||||
|
<value>javadoc</value>
|
||||||
|
</property>
|
||||||
|
</activation>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-javadoc-plugin</artifactId>
|
||||||
|
<version>2.9.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>attach-javadocs</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>jar</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<aggregate>true</aggregate>
|
||||||
|
<show>public</show>
|
||||||
|
<nohelp>true</nohelp>
|
||||||
|
<header>${project.artifactId}-${project.version}</header>
|
||||||
|
<footer>${project.artifactId}-${project.version}</footer>
|
||||||
|
<doctitle>${project.artifactId}-${project.version}</doctitle>
|
||||||
|
<links>
|
||||||
|
<link>https://github.com/alibaba/canal</link>
|
||||||
|
</links>
|
||||||
|
<outputDirectory>${project.build.directory}/apidocs/apidocs/${pom.version}</outputDirectory>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-scm-publish-plugin</artifactId>
|
||||||
|
<version>1.0-beta-2</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>attach-javadocs</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>publish-scm</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
|
||||||
|
<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}</checkinComment>
|
||||||
|
<content>${project.build.directory}/apidocs</content>
|
||||||
|
<skipDeletedFiles>true</skipDeletedFiles>
|
||||||
|
<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
|
||||||
|
<scmBranch>gh-pages</scmBranch>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
|
</project>
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
package com.alibaba.otter.canal.client;
|
||||||
|
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.protocol.Message;
|
||||||
|
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* canal数据操作客户端
|
||||||
|
*
|
||||||
|
* @author zebin.xuzb @ 2012-6-19
|
||||||
|
* @author jianghang
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public interface CanalConnector {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 链接对应的canal server
|
||||||
|
*
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
public void connect() throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 释放链接
|
||||||
|
*
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
public void disconnect() throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 检查下链接是否合法
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* 几种case下链接不合法:
|
||||||
|
* 1. 链接canal server失败，一直没有一个可用的链接，返回false
|
||||||
|
* 2. 当前客户端在进行running抢占的时候，做为备份节点存在，非处于工作节点，返回false
|
||||||
|
*
|
||||||
|
* 说明：
|
||||||
|
* a. 当前客户端一旦做为备份节点存在，当前所有的对{@linkplain CanalConnector}的操作都会处于阻塞状态，直到转为工作节点
|
||||||
|
* b. 所以业务方最好定时调用checkValid()方法用，比如调用CanalConnector所在线程的interrupt，直接退出CanalConnector，并根据自己的需要退出自己的资源
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
public boolean checkValid() throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端订阅，重复订阅时会更新对应的filter信息
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* 说明：
|
||||||
|
* a. 如果本次订阅中filter信息为空，则直接使用canal server服务端配置的filter信息
|
||||||
|
* b. 如果本次订阅中filter信息不为空，目前会直接替换canal server服务端配置的filter信息，以本次提交的为准
|
||||||
|
*
|
||||||
|
* TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @param clientIdentity
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
void subscribe(String filter) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端订阅，不提交客户端filter，以服务端的filter为准
|
||||||
|
*
|
||||||
|
* @param clientIdentity
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
void subscribe() throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 取消订阅
|
||||||
|
*
|
||||||
|
* @param clientIdentity
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
void unsubscribe() throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取数据，自动进行确认，该方法返回的条件：尝试拿batchSize条记录，有多少取多少，不会阻塞等待
|
||||||
|
*
|
||||||
|
* @param batchSize
|
||||||
|
* @return
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
Message get(int batchSize) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取数据，自动进行确认
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* 该方法返回的条件：
|
||||||
|
* a. 拿够batchSize条记录或者超过timeout时间
|
||||||
|
* b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @param batchSize
|
||||||
|
* @return
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 不指定 position 获取事件，该方法返回的条件: 尝试拿batchSize条记录，有多少取多少，不会阻塞等待<br/>
|
||||||
|
* canal 会记住此 client 最新的position。 <br/>
|
||||||
|
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
|
||||||
|
*
|
||||||
|
* @param batchSize
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
Message getWithoutAck(int batchSize) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 不指定 position 获取事件.
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* 该方法返回的条件：
|
||||||
|
* a. 拿够batchSize条记录或者超过timeout时间
|
||||||
|
* b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* canal 会记住此 client 最新的position。 <br/>
|
||||||
|
* 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
|
||||||
|
*
|
||||||
|
* @param batchSize
|
||||||
|
* @param timeout
|
||||||
|
* @param unit
|
||||||
|
* @return
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
|
||||||
|
*
|
||||||
|
* @param batchId
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
void ack(long batchId) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId
|
||||||
|
*
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
void rollback(long batchId) throws CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
|
||||||
|
*
|
||||||
|
* @throws CanalClientException
|
||||||
|
*/
|
||||||
|
void rollback() throws CanalClientException;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package com.alibaba.otter.canal.client;
|
||||||
|
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
|
||||||
|
import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
|
||||||
|
import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
|
||||||
|
import com.alibaba.otter.canal.client.impl.SimpleNodeAccessStrategy;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* canal connectors创建工具类
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-10-29 下午11:18:50
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class CanalConnectors {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建单链接的客户端链接
|
||||||
|
*
|
||||||
|
* @param address
|
||||||
|
* @param username
|
||||||
|
* @param password
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static CanalConnector newSingleConnector(SocketAddress address, String destination, String username,
|
||||||
|
String password) {
|
||||||
|
SimpleCanalConnector canalConnector = new SimpleCanalConnector(address, username, password, destination);
|
||||||
|
canalConnector.setSoTimeout(30 * 1000);
|
||||||
|
return canalConnector;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建带cluster模式的客户端链接，自动完成failover切换
|
||||||
|
*
|
||||||
|
* @param addresses
|
||||||
|
* @param username
|
||||||
|
* @param password
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
|
||||||
|
String username, String password) {
|
||||||
|
ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
|
||||||
|
new SimpleNodeAccessStrategy(addresses));
|
||||||
|
canalConnector.setSoTimeout(30 * 1000);
|
||||||
|
return canalConnector;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描
|
||||||
|
*
|
||||||
|
* @param username
|
||||||
|
* @param password
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
|
||||||
|
String password) {
|
||||||
|
ClusterCanalConnector canalConnector = new ClusterCanalConnector(
|
||||||
|
username,
|
||||||
|
password,
|
||||||
|
destination,
|
||||||
|
new ClusterNodeAccessStrategy(
|
||||||
|
destination,
|
||||||
|
ZkClientx.getZkClient(zkServers)));
|
||||||
|
canalConnector.setSoTimeout(30 * 1000);
|
||||||
|
return canalConnector;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package com.alibaba.otter.canal.client;
|
||||||
|
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 集群节点访问控制接口
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-10-29 下午07:55:41
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public interface CanalNodeAccessStrategy {
|
||||||
|
|
||||||
|
SocketAddress nextNode();
|
||||||
|
}
|
||||||
@@ -0,0 +1,327 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl;
|
||||||
|
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.exception.ExceptionUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.CanalConnector;
|
||||||
|
import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
|
||||||
|
import com.alibaba.otter.canal.protocol.Message;
|
||||||
|
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 集群版本connector实现，自带了failover功能<br/>
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-10-29 下午08:04:06
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ClusterCanalConnector implements CanalConnector {
|
||||||
|
|
||||||
|
private final Logger logger = LoggerFactory.getLogger(this.getClass());
|
||||||
|
private String username;
|
||||||
|
private String password;
|
||||||
|
private int soTimeout = 10000;
|
||||||
|
private int retryTimes = 3;
|
||||||
|
private int retryInterval = 5000; // 重试的时间间隔，默认5秒
|
||||||
|
private CanalNodeAccessStrategy accessStrategy;
|
||||||
|
private SimpleCanalConnector currentConnector;
|
||||||
|
private String destination;
|
||||||
|
private String filter; // 记录上一次的filter提交值,便于自动重试时提交
|
||||||
|
|
||||||
|
public ClusterCanalConnector(String username, String password, String destination,
|
||||||
|
CanalNodeAccessStrategy accessStrategy){
|
||||||
|
this.username = username;
|
||||||
|
this.password = password;
|
||||||
|
this.destination = destination;
|
||||||
|
this.accessStrategy = accessStrategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void connect() throws CanalClientException {
|
||||||
|
while (currentConnector == null) {
|
||||||
|
SocketAddress nextAddress = this.accessStrategy.nextNode();
|
||||||
|
int times = 0;
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination);
|
||||||
|
currentConnector.setSoTimeout(soTimeout);
|
||||||
|
if (filter != null) {
|
||||||
|
currentConnector.setFilter(filter);
|
||||||
|
}
|
||||||
|
if (accessStrategy instanceof ClusterNodeAccessStrategy) {
|
||||||
|
currentConnector.setZkClientx(((ClusterNodeAccessStrategy) accessStrategy).getZkClient());
|
||||||
|
}
|
||||||
|
|
||||||
|
currentConnector.connect();
|
||||||
|
break;
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.warn("failed to connect to:{} after retry {} times", nextAddress, times);
|
||||||
|
currentConnector.disconnect();
|
||||||
|
currentConnector = null;
|
||||||
|
// retry for #retryTimes for each node when trying to
|
||||||
|
// connect to it.
|
||||||
|
times = times + 1;
|
||||||
|
if (times >= retryTimes) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
} else {
|
||||||
|
// fixed issue #55，增加sleep控制，避免重试connect时cpu使用过高
|
||||||
|
try {
|
||||||
|
Thread.sleep(retryInterval);
|
||||||
|
} catch (InterruptedException e1) {
|
||||||
|
throw new CanalClientException(e1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean checkValid() {
|
||||||
|
return currentConnector != null && currentConnector.checkValid();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void disconnect() throws CanalClientException {
|
||||||
|
if (currentConnector != null) {
|
||||||
|
currentConnector.disconnect();
|
||||||
|
currentConnector = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void subscribe() throws CanalClientException {
|
||||||
|
subscribe(""); // 传递空字符即可
|
||||||
|
}
|
||||||
|
|
||||||
|
public void subscribe(String filter) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
currentConnector.subscribe(filter);
|
||||||
|
this.filter = filter;
|
||||||
|
return;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when subscribing from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new CanalClientException("failed to subscribe after " + times + " times retry.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void unsubscribe() throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
currentConnector.unsubscribe();
|
||||||
|
return;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CanalClientException("failed to unsubscribe after " + times + " times retry.");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message get(int batchSize) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
Message msg = currentConnector.get(batchSize);
|
||||||
|
return msg;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when getting data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
Message msg = currentConnector.get(batchSize, timeout, unit);
|
||||||
|
return msg;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when getting data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message getWithoutAck(int batchSize) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
Message msg = currentConnector.getWithoutAck(batchSize);
|
||||||
|
return msg;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
|
||||||
|
return msg;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CanalClientException("failed to fetch the data after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void rollback(long batchId) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
currentConnector.rollback(batchId);
|
||||||
|
return;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CanalClientException("failed to rollback after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void rollback() throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
currentConnector.rollback();
|
||||||
|
return;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new CanalClientException("failed to rollback after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ack(long batchId) throws CanalClientException {
|
||||||
|
int times = 0;
|
||||||
|
while (times < retryTimes) {
|
||||||
|
try {
|
||||||
|
currentConnector.ack(batchId);
|
||||||
|
return;
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("something goes wrong when acking data from server:{}\n{}",
|
||||||
|
currentConnector.getAddress(),
|
||||||
|
ExceptionUtils.getFullStackTrace(t));
|
||||||
|
times++;
|
||||||
|
restart();
|
||||||
|
logger.info("restart the connector for next round retry.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new CanalClientException("failed to ack after " + times + " times retry");
|
||||||
|
}
|
||||||
|
|
||||||
|
private void restart() throws CanalClientException {
|
||||||
|
disconnect();
|
||||||
|
try {
|
||||||
|
Thread.sleep(retryInterval);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
connect();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================= setter / getter
|
||||||
|
// ============================
|
||||||
|
|
||||||
|
public String getUsername() {
|
||||||
|
return username;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUsername(String username) {
|
||||||
|
this.username = username;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPassword() {
|
||||||
|
return password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPassword(String password) {
|
||||||
|
this.password = password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getSoTimeout() {
|
||||||
|
return soTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSoTimeout(int soTimeout) {
|
||||||
|
this.soTimeout = soTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getRetryTimes() {
|
||||||
|
return retryTimes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetryTimes(int retryTimes) {
|
||||||
|
this.retryTimes = retryTimes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getRetryInterval() {
|
||||||
|
return retryInterval;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetryInterval(int retryInterval) {
|
||||||
|
this.retryInterval = retryInterval;
|
||||||
|
}
|
||||||
|
|
||||||
|
public CanalNodeAccessStrategy getAccessStrategy() {
|
||||||
|
return accessStrategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAccessStrategy(CanalNodeAccessStrategy accessStrategy) {
|
||||||
|
this.accessStrategy = accessStrategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
public SimpleCanalConnector getCurrentConnector() {
|
||||||
|
return currentConnector;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+112
@@ -0,0 +1,112 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl;
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.IZkChildListener;
|
||||||
|
import org.I0Itec.zkclient.IZkDataListener;
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
|
||||||
|
import com.alibaba.otter.canal.common.utils.JsonUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
|
||||||
|
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 集群模式的调度策略
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-12-3 下午10:01:04
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
|
||||||
|
|
||||||
|
private IZkChildListener childListener; // 监听所有的服务器列表
|
||||||
|
private IZkDataListener dataListener; // 监听当前的工作节点
|
||||||
|
private ZkClientx zkClient;
|
||||||
|
private volatile List<InetSocketAddress> currentAddress = new ArrayList<InetSocketAddress>();
|
||||||
|
private volatile InetSocketAddress runningAddress = null;
|
||||||
|
|
||||||
|
public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){
|
||||||
|
this.zkClient = zkClient;
|
||||||
|
childListener = new IZkChildListener() {
|
||||||
|
|
||||||
|
public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
|
||||||
|
initClusters(currentChilds);
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
dataListener = new IZkDataListener() {
|
||||||
|
|
||||||
|
public void handleDataDeleted(String dataPath) throws Exception {
|
||||||
|
runningAddress = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleDataChange(String dataPath, Object data) throws Exception {
|
||||||
|
initRunning(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
String clusterPath = ZookeeperPathUtils.getDestinationClusterRoot(destination);
|
||||||
|
this.zkClient.subscribeChildChanges(clusterPath, childListener);
|
||||||
|
initClusters(this.zkClient.getChildren(clusterPath));
|
||||||
|
|
||||||
|
String runningPath = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||||
|
this.zkClient.subscribeDataChanges(runningPath, dataListener);
|
||||||
|
initRunning(this.zkClient.readData(runningPath, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
public SocketAddress nextNode() {
|
||||||
|
if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点
|
||||||
|
return runningAddress;
|
||||||
|
} else if (!currentAddress.isEmpty()) { // 如果不存在已经启动的服务，可能服务是一种lazy启动，随机选择一台触发服务器进行启动
|
||||||
|
return currentAddress.get(0);// 默认返回第一个节点，之前已经做过shuffle
|
||||||
|
} else {
|
||||||
|
throw new CanalClientException("no alive canal server");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initClusters(List<String> currentChilds) {
|
||||||
|
if (currentChilds == null || currentChilds.isEmpty()) {
|
||||||
|
currentAddress = new ArrayList<InetSocketAddress>();
|
||||||
|
} else {
|
||||||
|
List<InetSocketAddress> addresses = new ArrayList<InetSocketAddress>();
|
||||||
|
for (String address : currentChilds) {
|
||||||
|
String[] strs = StringUtils.split(address, ":");
|
||||||
|
if (strs != null && strs.length == 2) {
|
||||||
|
addresses.add(new InetSocketAddress(strs[0], Integer.valueOf(strs[1])));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Collections.shuffle(addresses);
|
||||||
|
currentAddress = addresses;// 直接切换引用
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initRunning(Object data) {
|
||||||
|
if (data == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
|
||||||
|
String[] strs = StringUtils.split(runningData.getAddress(), ':');
|
||||||
|
if (strs.length == 2) {
|
||||||
|
runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setZkClient(ZkClientx zkClient) {
|
||||||
|
this.zkClient = zkClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZkClientx getZkClient() {
|
||||||
|
return zkClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,475 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.ByteOrder;
|
||||||
|
import java.nio.channels.SocketChannel;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.CanalConnector;
|
||||||
|
import com.alibaba.otter.canal.client.impl.running.ClientRunningData;
|
||||||
|
import com.alibaba.otter.canal.client.impl.running.ClientRunningListener;
|
||||||
|
import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor;
|
||||||
|
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||||
|
import com.alibaba.otter.canal.common.utils.BooleanMutex;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Compression;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Get;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
|
||||||
|
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
|
||||||
|
import com.alibaba.otter.canal.protocol.ClientIdentity;
|
||||||
|
import com.alibaba.otter.canal.protocol.Message;
|
||||||
|
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
|
||||||
|
import com.google.protobuf.ByteString;
|
||||||
|
import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-10-24 下午05:37:20
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class SimpleCanalConnector implements CanalConnector {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(SimpleCanalConnector.class);
|
||||||
|
private SocketAddress address;
|
||||||
|
private String username;
|
||||||
|
private String password;
|
||||||
|
private int soTimeout = 60000; // milliseconds
|
||||||
|
private String filter; // 记录上一次的filter提交值,便于自动重试时提交
|
||||||
|
|
||||||
|
private final ByteBuffer readHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
|
||||||
|
private final ByteBuffer writeHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
|
||||||
|
private SocketChannel channel;
|
||||||
|
private List<Compression> supportedCompressions = new ArrayList<Compression>();
|
||||||
|
private ClientIdentity clientIdentity;
|
||||||
|
private ClientRunningMonitor runningMonitor; // 运行控制
|
||||||
|
private ZkClientx zkClientx;
|
||||||
|
private BooleanMutex mutex = new BooleanMutex(false);
|
||||||
|
private volatile boolean connected = false; // 代表connected是否已正常执行，因为有HA，不代表在工作中
|
||||||
|
private boolean rollbackOnConnect = true; // 是否在connect链接成功后，自动执行rollback操作
|
||||||
|
private boolean rollbackOnDisConnect = false; // 是否在connect链接成功后，自动执行rollback操作
|
||||||
|
|
||||||
|
// 读写数据分别使用不同的锁进行控制，减小锁粒度,读也需要排他锁，并发度容易造成数据包混乱，反序列化失败
|
||||||
|
private Object readDataLock = new Object();
|
||||||
|
private Object writeDataLock = new Object();
|
||||||
|
|
||||||
|
public SimpleCanalConnector(SocketAddress address, String username, String password, String destination){
|
||||||
|
this(address, username, password, destination, 60000);
|
||||||
|
}
|
||||||
|
|
||||||
|
public SimpleCanalConnector(SocketAddress address, String username, String password, String destination,
|
||||||
|
int soTimeout){
|
||||||
|
this.address = address;
|
||||||
|
this.username = username;
|
||||||
|
this.password = password;
|
||||||
|
this.soTimeout = soTimeout;
|
||||||
|
this.clientIdentity = new ClientIdentity(destination, (short) 1001);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void connect() throws CanalClientException {
|
||||||
|
if (connected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (runningMonitor != null) {
|
||||||
|
if (!runningMonitor.isStart()) {
|
||||||
|
runningMonitor.start();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
waitClientRunning();
|
||||||
|
doConnect();
|
||||||
|
if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
|
||||||
|
subscribe(filter);
|
||||||
|
}
|
||||||
|
if (rollbackOnConnect) {
|
||||||
|
rollback();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
connected = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void disconnect() throws CanalClientException {
|
||||||
|
if (rollbackOnDisConnect && channel.isConnected()) {
|
||||||
|
rollback();
|
||||||
|
}
|
||||||
|
|
||||||
|
connected = false;
|
||||||
|
if (runningMonitor != null) {
|
||||||
|
if (runningMonitor.isStart()) {
|
||||||
|
runningMonitor.stop();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
doDisconnnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private InetSocketAddress doConnect() throws CanalClientException {
|
||||||
|
try {
|
||||||
|
channel = SocketChannel.open();
|
||||||
|
channel.socket().setSoTimeout(soTimeout);
|
||||||
|
channel.connect(address);
|
||||||
|
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||||
|
if (p.getVersion() != 1) {
|
||||||
|
throw new CanalClientException("unsupported version at this client.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (p.getType() != PacketType.HANDSHAKE) {
|
||||||
|
throw new CanalClientException("expect handshake but found other type.");
|
||||||
|
}
|
||||||
|
//
|
||||||
|
Handshake handshake = Handshake.parseFrom(p.getBody());
|
||||||
|
supportedCompressions.addAll(handshake.getSupportedCompressionsList());
|
||||||
|
//
|
||||||
|
ClientAuth ca = ClientAuth.newBuilder()
|
||||||
|
.setUsername(username != null ? username : "")
|
||||||
|
.setNetReadTimeout(soTimeout)
|
||||||
|
.setNetWriteTimeout(soTimeout)
|
||||||
|
.build();
|
||||||
|
writeWithHeader(channel,
|
||||||
|
Packet.newBuilder()
|
||||||
|
.setType(PacketType.CLIENTAUTHENTICATION)
|
||||||
|
.setBody(ca.toByteString())
|
||||||
|
.build()
|
||||||
|
.toByteArray());
|
||||||
|
//
|
||||||
|
Packet ack = Packet.parseFrom(readNextPacket(channel));
|
||||||
|
if (ack.getType() != PacketType.ACK) {
|
||||||
|
throw new CanalClientException("unexpected packet type when ack is expected");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ack ackBody = Ack.parseFrom(ack.getBody());
|
||||||
|
if (ackBody.getErrorCode() > 0) {
|
||||||
|
throw new CanalClientException("something goes wrong when doing authentication: "
|
||||||
|
+ ackBody.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
connected = true;
|
||||||
|
return new InetSocketAddress(channel.socket().getLocalAddress(), channel.socket().getLocalPort());
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doDisconnnect() throws CanalClientException {
|
||||||
|
if (channel != null) {
|
||||||
|
try {
|
||||||
|
channel.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.warn("exception on closing channel:{} \n {}", channel, e);
|
||||||
|
}
|
||||||
|
channel = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void subscribe() throws CanalClientException {
|
||||||
|
subscribe(""); // 传递空字符即可
|
||||||
|
}
|
||||||
|
|
||||||
|
public void subscribe(String filter) throws CanalClientException {
|
||||||
|
waitClientRunning();
|
||||||
|
try {
|
||||||
|
writeWithHeader(channel,
|
||||||
|
Packet.newBuilder()
|
||||||
|
.setType(PacketType.SUBSCRIPTION)
|
||||||
|
.setBody(Sub.newBuilder()
|
||||||
|
.setDestination(clientIdentity.getDestination())
|
||||||
|
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||||
|
.setFilter(filter != null ? filter : "")
|
||||||
|
.build()
|
||||||
|
.toByteString())
|
||||||
|
.build()
|
||||||
|
.toByteArray());
|
||||||
|
//
|
||||||
|
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||||
|
Ack ack = Ack.parseFrom(p.getBody());
|
||||||
|
if (ack.getErrorCode() > 0) {
|
||||||
|
throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
clientIdentity.setFilter(filter);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void unsubscribe() throws CanalClientException {
|
||||||
|
waitClientRunning();
|
||||||
|
try {
|
||||||
|
writeWithHeader(channel,
|
||||||
|
Packet.newBuilder()
|
||||||
|
.setType(PacketType.UNSUBSCRIPTION)
|
||||||
|
.setBody(Unsub.newBuilder()
|
||||||
|
.setDestination(clientIdentity.getDestination())
|
||||||
|
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||||
|
.build()
|
||||||
|
.toByteString())
|
||||||
|
.build()
|
||||||
|
.toByteArray());
|
||||||
|
//
|
||||||
|
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||||
|
Ack ack = Ack.parseFrom(p.getBody());
|
||||||
|
if (ack.getErrorCode() > 0) {
|
||||||
|
throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message get(int batchSize) throws CanalClientException {
|
||||||
|
return get(batchSize, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
|
Message message = getWithoutAck(batchSize, timeout, unit);
|
||||||
|
ack(message.getId());
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message getWithoutAck(int batchSize) throws CanalClientException {
|
||||||
|
return getWithoutAck(batchSize, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
|
||||||
|
waitClientRunning();
|
||||||
|
try {
|
||||||
|
int size = (batchSize <= 0) ? 1000 : batchSize;
|
||||||
|
long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制
|
||||||
|
if (unit == null) {
|
||||||
|
unit = TimeUnit.MILLISECONDS;
|
||||||
|
}
|
||||||
|
|
||||||
|
writeWithHeader(channel,
|
||||||
|
Packet.newBuilder()
|
||||||
|
.setType(PacketType.GET)
|
||||||
|
.setBody(Get.newBuilder()
|
||||||
|
.setAutoAck(false)
|
||||||
|
.setDestination(clientIdentity.getDestination())
|
||||||
|
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||||
|
.setFetchSize(size)
|
||||||
|
.setTimeout(time)
|
||||||
|
.setUnit(unit.ordinal())
|
||||||
|
.build()
|
||||||
|
.toByteString())
|
||||||
|
.build()
|
||||||
|
.toByteArray());
|
||||||
|
|
||||||
|
return receiveMessages();
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Message receiveMessages() throws InvalidProtocolBufferException, IOException {
|
||||||
|
Packet p = Packet.parseFrom(readNextPacket(channel));
|
||||||
|
switch (p.getType()) {
|
||||||
|
case MESSAGES: {
|
||||||
|
if (!p.getCompression().equals(Compression.NONE)) {
|
||||||
|
throw new CanalClientException("compression is not supported in this connector");
|
||||||
|
}
|
||||||
|
|
||||||
|
Messages messages = Messages.parseFrom(p.getBody());
|
||||||
|
Message result = new Message(messages.getBatchId());
|
||||||
|
for (ByteString byteString : messages.getMessagesList()) {
|
||||||
|
result.addEntry(Entry.parseFrom(byteString));
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
case ACK: {
|
||||||
|
Ack ack = Ack.parseFrom(p.getBody());
|
||||||
|
throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
|
||||||
|
}
|
||||||
|
default: {
|
||||||
|
throw new CanalClientException("unexpected packet type: " + p.getType());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ack(long batchId) throws CanalClientException {
|
||||||
|
waitClientRunning();
|
||||||
|
ClientAck ca = ClientAck.newBuilder()
|
||||||
|
.setDestination(clientIdentity.getDestination())
|
||||||
|
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||||
|
.setBatchId(batchId)
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
writeWithHeader(channel, Packet.newBuilder()
|
||||||
|
.setType(PacketType.CLIENTACK)
|
||||||
|
.setBody(ca.toByteString())
|
||||||
|
.build()
|
||||||
|
.toByteArray());
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void rollback(long batchId) throws CanalClientException {
|
||||||
|
waitClientRunning();
|
||||||
|
ClientRollback ca = ClientRollback.newBuilder()
|
||||||
|
.setDestination(clientIdentity.getDestination())
|
||||||
|
.setClientId(String.valueOf(clientIdentity.getClientId()))
|
||||||
|
.setBatchId(batchId)
|
||||||
|
.build();
|
||||||
|
try {
|
||||||
|
writeWithHeader(channel, Packet.newBuilder()
|
||||||
|
.setType(PacketType.CLIENTROLLBACK)
|
||||||
|
.setBody(ca.toByteString())
|
||||||
|
.build()
|
||||||
|
.toByteArray());
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void rollback() throws CanalClientException {
|
||||||
|
waitClientRunning();
|
||||||
|
rollback(0);// 0代笔未设置
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== helper method ====================
|
||||||
|
|
||||||
|
private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
|
||||||
|
synchronized (writeDataLock) {
|
||||||
|
writeHeader.clear();
|
||||||
|
writeHeader.putInt(body.length);
|
||||||
|
writeHeader.flip();
|
||||||
|
channel.write(writeHeader);
|
||||||
|
channel.write(ByteBuffer.wrap(body));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private byte[] readNextPacket(SocketChannel channel) throws IOException {
|
||||||
|
synchronized (readDataLock) {
|
||||||
|
readHeader.clear();
|
||||||
|
read(channel, readHeader);
|
||||||
|
int bodyLen = readHeader.getInt(0);
|
||||||
|
ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen).order(ByteOrder.BIG_ENDIAN);
|
||||||
|
read(channel, bodyBuf);
|
||||||
|
return bodyBuf.array();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
|
||||||
|
while (buffer.hasRemaining()) {
|
||||||
|
int r = channel.read(buffer);
|
||||||
|
if (r == -1) {
|
||||||
|
throw new IOException("end of stream when reading header");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private synchronized void initClientRunningMonitor(ClientIdentity clientIdentity) {
|
||||||
|
if (zkClientx != null && clientIdentity != null && runningMonitor == null) {
|
||||||
|
ClientRunningData clientData = new ClientRunningData();
|
||||||
|
clientData.setClientId(clientIdentity.getClientId());
|
||||||
|
clientData.setAddress(AddressUtils.getHostIp());
|
||||||
|
|
||||||
|
runningMonitor = new ClientRunningMonitor();
|
||||||
|
runningMonitor.setDestination(clientIdentity.getDestination());
|
||||||
|
runningMonitor.setZkClient(zkClientx);
|
||||||
|
runningMonitor.setClientData(clientData);
|
||||||
|
runningMonitor.setListener(new ClientRunningListener() {
|
||||||
|
|
||||||
|
public InetSocketAddress processActiveEnter() {
|
||||||
|
InetSocketAddress address = doConnect();
|
||||||
|
mutex.set(true);
|
||||||
|
if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
|
||||||
|
subscribe(filter);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rollbackOnConnect) {
|
||||||
|
rollback();
|
||||||
|
}
|
||||||
|
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processActiveExit() {
|
||||||
|
mutex.set(false);
|
||||||
|
doDisconnnect();
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void waitClientRunning() {
|
||||||
|
try {
|
||||||
|
if (zkClientx != null) {
|
||||||
|
if (!connected) {// 未调用connect
|
||||||
|
throw new CanalClientException("should connect first");
|
||||||
|
}
|
||||||
|
|
||||||
|
mutex.get();// 阻塞等待
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new CanalClientException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean checkValid() {
|
||||||
|
if (zkClientx != null) {
|
||||||
|
return mutex.state();
|
||||||
|
} else {
|
||||||
|
return true;// 默认都放过
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public SocketAddress getAddress() {
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUsername() {
|
||||||
|
return username;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPassword() {
|
||||||
|
return password;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getSoTimeout() {
|
||||||
|
return soTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSoTimeout(int soTimeout) {
|
||||||
|
this.soTimeout = soTimeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setZkClientx(ZkClientx zkClientx) {
|
||||||
|
this.zkClientx = zkClientx;
|
||||||
|
initClientRunningMonitor(this.clientIdentity);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRollbackOnConnect(boolean rollbackOnConnect) {
|
||||||
|
this.rollbackOnConnect = rollbackOnConnect;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRollbackOnDisConnect(boolean rollbackOnDisConnect) {
|
||||||
|
this.rollbackOnDisConnect = rollbackOnDisConnect;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFilter(String filter) {
|
||||||
|
this.filter = filter;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+35
@@ -0,0 +1,35 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl;
|
||||||
|
|
||||||
|
import java.net.SocketAddress;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 简单版本的node访问实现
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-10-29 下午08:00:23
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
|
||||||
|
|
||||||
|
private List<SocketAddress> nodes = new ArrayList<SocketAddress>();
|
||||||
|
private int index = 0;
|
||||||
|
|
||||||
|
public SimpleNodeAccessStrategy(List<? extends SocketAddress> nodes){
|
||||||
|
if (nodes == null || nodes.size() < 1) {
|
||||||
|
throw new IllegalArgumentException("at least 1 node required.");
|
||||||
|
}
|
||||||
|
this.nodes.addAll(nodes);
|
||||||
|
}
|
||||||
|
|
||||||
|
public SocketAddress nextNode() {
|
||||||
|
try {
|
||||||
|
return nodes.get(index);
|
||||||
|
} finally {
|
||||||
|
index = (index + 1) % nodes.size();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+39
@@ -0,0 +1,39 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl.running;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* client running状态信息
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-11-22 下午03:41:50
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ClientRunningData {
|
||||||
|
|
||||||
|
private short clientId;
|
||||||
|
private String address;
|
||||||
|
private boolean active = true;
|
||||||
|
|
||||||
|
public short getClientId() {
|
||||||
|
return clientId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setClientId(short clientId) {
|
||||||
|
this.clientId = clientId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAddress() {
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAddress(String address) {
|
||||||
|
this.address = address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isActive() {
|
||||||
|
return active;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setActive(boolean active) {
|
||||||
|
this.active = active;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+23
@@ -0,0 +1,23 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl.running;
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 触发一下mainstem发生切换
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-9-11 下午02:26:03
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public interface ClientRunningListener {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 触发现在轮到自己做为active，需要载入上一个active的上下文数据
|
||||||
|
*/
|
||||||
|
public InetSocketAddress processActiveEnter();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 触发一下当前active模式失败
|
||||||
|
*/
|
||||||
|
public void processActiveExit();
|
||||||
|
|
||||||
|
}
|
||||||
+228
@@ -0,0 +1,228 @@
|
|||||||
|
package com.alibaba.otter.canal.client.impl.running;
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.IZkDataListener;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkInterruptedException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNoNodeException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
|
||||||
|
import org.apache.zookeeper.CreateMode;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.slf4j.MDC;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||||
|
import com.alibaba.otter.canal.common.utils.BooleanMutex;
|
||||||
|
import com.alibaba.otter.canal.common.utils.JsonUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* clinet running控制
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-11-22 下午03:43:01
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ClientRunningMonitor extends AbstractCanalLifeCycle {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ClientRunningMonitor.class);
|
||||||
|
private ZkClientx zkClient;
|
||||||
|
private String destination;
|
||||||
|
private ClientRunningData clientData;
|
||||||
|
private IZkDataListener dataListener;
|
||||||
|
private BooleanMutex mutex = new BooleanMutex(false);
|
||||||
|
private volatile boolean release = false;
|
||||||
|
private volatile ClientRunningData activeData;
|
||||||
|
private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1);
|
||||||
|
private ClientRunningListener listener;
|
||||||
|
private int delayTime = 5;
|
||||||
|
|
||||||
|
public ClientRunningMonitor(){
|
||||||
|
dataListener = new IZkDataListener() {
|
||||||
|
|
||||||
|
public void handleDataChange(String dataPath, Object data) throws Exception {
|
||||||
|
MDC.put("destination", destination);
|
||||||
|
ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class);
|
||||||
|
if (!isMine(runningData.getAddress())) {
|
||||||
|
mutex.set(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
|
||||||
|
release = true;
|
||||||
|
releaseRunning();// 彻底释放mainstem
|
||||||
|
}
|
||||||
|
|
||||||
|
activeData = (ClientRunningData) runningData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleDataDeleted(String dataPath) throws Exception {
|
||||||
|
MDC.put("destination", destination);
|
||||||
|
mutex.set(false);
|
||||||
|
if (!release && activeData != null && isMine(activeData.getAddress())) {
|
||||||
|
// 如果上一次active的状态就是本机，则即时触发一下active抢占
|
||||||
|
initRunning();
|
||||||
|
} else {
|
||||||
|
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
|
||||||
|
delayExector.schedule(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
initRunning();
|
||||||
|
}
|
||||||
|
}, delayTime, TimeUnit.SECONDS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start() {
|
||||||
|
super.start();
|
||||||
|
|
||||||
|
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||||
|
zkClient.subscribeDataChanges(path, dataListener);
|
||||||
|
initRunning();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
super.stop();
|
||||||
|
|
||||||
|
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||||
|
zkClient.unsubscribeDataChanges(path, dataListener);
|
||||||
|
releaseRunning(); // 尝试一下release
|
||||||
|
}
|
||||||
|
|
||||||
|
public void initRunning() {
|
||||||
|
if (!isStart()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||||
|
// 序列化
|
||||||
|
byte[] bytes = JsonUtils.marshalToByte(clientData);
|
||||||
|
try {
|
||||||
|
mutex.set(false);
|
||||||
|
zkClient.create(path, bytes, CreateMode.EPHEMERAL);
|
||||||
|
processActiveEnter();// 触发一下事件
|
||||||
|
activeData = clientData;
|
||||||
|
mutex.set(true);
|
||||||
|
} catch (ZkNodeExistsException e) {
|
||||||
|
bytes = zkClient.readData(path, true);
|
||||||
|
if (bytes == null) {// 如果不存在节点，立即尝试一次
|
||||||
|
initRunning();
|
||||||
|
} else {
|
||||||
|
activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
|
||||||
|
}
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()),
|
||||||
|
true); // 尝试创建父节点
|
||||||
|
initRunning();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 阻塞等待自己成为active，如果自己成为active，立马返回
|
||||||
|
*
|
||||||
|
* @throws InterruptedException
|
||||||
|
*/
|
||||||
|
public void waitForActive() throws InterruptedException {
|
||||||
|
initRunning();
|
||||||
|
mutex.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 检查当前的状态
|
||||||
|
*/
|
||||||
|
public boolean check() {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||||
|
try {
|
||||||
|
byte[] bytes = zkClient.readData(path);
|
||||||
|
ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
|
||||||
|
activeData = eventData;// 更新下为最新值
|
||||||
|
// 检查下nid是否为自己
|
||||||
|
boolean result = isMine(activeData.getAddress());
|
||||||
|
if (!result) {
|
||||||
|
logger.warn("canal is running in [{}] , but not in [{}]",
|
||||||
|
activeData.getAddress(),
|
||||||
|
clientData.getAddress());
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
logger.warn("canal is not run any in node");
|
||||||
|
return false;
|
||||||
|
} catch (ZkInterruptedException e) {
|
||||||
|
logger.warn("canal check is interrupt");
|
||||||
|
Thread.interrupted();// 清除interrupt标记
|
||||||
|
return check();
|
||||||
|
} catch (ZkException e) {
|
||||||
|
logger.warn("canal check is failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean releaseRunning() {
|
||||||
|
if (check()) {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
|
||||||
|
zkClient.delete(path);
|
||||||
|
mutex.set(false);
|
||||||
|
processActiveExit();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================== helper method ======================
|
||||||
|
|
||||||
|
private boolean isMine(String address) {
|
||||||
|
return address.equals(clientData.getAddress());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processActiveEnter() {
|
||||||
|
if (listener != null) {
|
||||||
|
// 触发回调，建立与server的socket链接
|
||||||
|
InetSocketAddress connectAddress = listener.processActiveEnter();
|
||||||
|
String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
|
||||||
|
this.clientData.setAddress(address);
|
||||||
|
|
||||||
|
String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
|
||||||
|
this.clientData.getClientId());
|
||||||
|
// 序列化
|
||||||
|
byte[] bytes = JsonUtils.marshalToByte(clientData);
|
||||||
|
zkClient.writeData(path, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processActiveExit() {
|
||||||
|
if (listener != null) {
|
||||||
|
listener.processActiveExit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setListener(ClientRunningListener listener) {
|
||||||
|
this.listener = listener;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===================== setter / getter =======================
|
||||||
|
|
||||||
|
public void setDestination(String destination) {
|
||||||
|
this.destination = destination;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setClientData(ClientRunningData clientData) {
|
||||||
|
this.clientData = clientData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDelayTime(int delayTime) {
|
||||||
|
this.delayTime = delayTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setZkClient(ZkClientx zkClient) {
|
||||||
|
this.zkClient = zkClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package com.alibaba.otter.canal.client.running;
|
||||||
|
|
||||||
|
import org.junit.Assert;
|
||||||
|
|
||||||
|
public class AbstractZkTest {
|
||||||
|
|
||||||
|
protected String destination = "ljhtest1";
|
||||||
|
protected String cluster1 = "127.0.0.1:2188";
|
||||||
|
protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188";
|
||||||
|
|
||||||
|
public void sleep(long time) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(time);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Assert.fail(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
package com.alibaba.otter.canal.client.running;
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.math.RandomUtils;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.client.impl.running.ClientRunningData;
|
||||||
|
import com.alibaba.otter.canal.client.impl.running.ClientRunningListener;
|
||||||
|
import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor;
|
||||||
|
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||||
|
|
||||||
|
public class ClientRunningTest extends AbstractZkTest {
|
||||||
|
|
||||||
|
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||||
|
private short clientId = 1001;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||||
|
zkclientx.deleteRecursive(path);
|
||||||
|
|
||||||
|
zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||||
|
zkclientx.deleteRecursive(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOneServer() {
|
||||||
|
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||||
|
ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
|
||||||
|
runningMonitor.start();
|
||||||
|
sleep(2000L);
|
||||||
|
runningMonitor.stop();
|
||||||
|
sleep(2000L);
|
||||||
|
|
||||||
|
if (countLatch.getCount() != 0) {
|
||||||
|
Assert.fail();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMultiServer() {
|
||||||
|
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||||
|
final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
|
||||||
|
final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
|
||||||
|
final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
|
||||||
|
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||||
|
executor.submit(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
if (!runningMonitor1.isStart()) {
|
||||||
|
runningMonitor1.start();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
runningMonitor1.stop();
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
executor.submit(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
if (!runningMonitor2.isStart()) {
|
||||||
|
runningMonitor2.start();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
runningMonitor2.stop();
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
executor.submit(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
if (!runningMonitor3.isStart()) {
|
||||||
|
runningMonitor3.start();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
runningMonitor3.stop();
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
sleep(30000L);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
|
||||||
|
final int port) {
|
||||||
|
ClientRunningData clientData = new ClientRunningData();
|
||||||
|
clientData.setClientId(clientId);
|
||||||
|
clientData.setAddress(AddressUtils.getHostIp());
|
||||||
|
|
||||||
|
ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
|
||||||
|
runningMonitor.setDestination(destination);
|
||||||
|
runningMonitor.setZkClient(zkclientx);
|
||||||
|
runningMonitor.setClientData(clientData);
|
||||||
|
runningMonitor.setListener(new ClientRunningListener() {
|
||||||
|
|
||||||
|
public InetSocketAddress processActiveEnter() {
|
||||||
|
System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
|
||||||
|
countLatch.countDown();
|
||||||
|
return new InetSocketAddress(AddressUtils.getHostIp(), port);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processActiveExit() {
|
||||||
|
countLatch.countDown();
|
||||||
|
System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
runningMonitor.setDelayTime(1);
|
||||||
|
return runningMonitor;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
<configuration scan="true" scanPeriod=" 5 seconds">
|
||||||
|
|
||||||
|
<jmxConfigurator />
|
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
|
||||||
|
</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<root level="WARN">
|
||||||
|
<appender-ref ref="STDOUT"/>
|
||||||
|
</root>
|
||||||
|
</configuration>
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal</artifactId>
|
||||||
|
<version>1.0.19-SNAPSHOT</version>
|
||||||
|
<relativePath>../pom.xml</relativePath>
|
||||||
|
</parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal.common</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
<name>canal common module for otter ${project.version}</name>
|
||||||
|
<url>http://b2b-doc.alibaba-inc.com/display/opentech/Otter</url>
|
||||||
|
<dependencies>
|
||||||
|
<!-- zk -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.zookeeper</groupId>
|
||||||
|
<artifactId>zookeeper</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.github.sgroschupf</groupId>
|
||||||
|
<artifactId>zkclient</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<!-- external -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>commons-io</groupId>
|
||||||
|
<artifactId>commons-io</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>commons-lang</groupId>
|
||||||
|
<artifactId>commons-lang</artifactId>
|
||||||
|
<version>2.6</version>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.springframework</groupId>
|
||||||
|
<artifactId>spring</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba</groupId>
|
||||||
|
<artifactId>fastjson</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.google.guava</groupId>
|
||||||
|
<artifactId>guava</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<!-- log -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-core</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-classic</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>jcl-over-slf4j</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<!-- junit -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</project>
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.alibaba.otter.canal.common;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 基本实现
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-7-12 上午10:11:07
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public abstract class AbstractCanalLifeCycle implements CanalLifeCycle {
|
||||||
|
|
||||||
|
protected volatile boolean running = false; // 是否处于运行中
|
||||||
|
|
||||||
|
public boolean isStart() {
|
||||||
|
return running;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start() {
|
||||||
|
if (running) {
|
||||||
|
throw new CanalException(this.getClass().getName() + " has startup , don't repeat start");
|
||||||
|
}
|
||||||
|
|
||||||
|
running = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
if (!running) {
|
||||||
|
throw new CanalException(this.getClass().getName() + " isn't start , please check");
|
||||||
|
}
|
||||||
|
|
||||||
|
running = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package com.alibaba.otter.canal.common;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.exception.NestableRuntimeException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author jianghang 2012-7-12 上午10:10:31
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class CanalException extends NestableRuntimeException {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = -654893533794556357L;
|
||||||
|
|
||||||
|
public CanalException(String errorCode){
|
||||||
|
super(errorCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CanalException(String errorCode, Throwable cause){
|
||||||
|
super(errorCode, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CanalException(String errorCode, String errorDesc){
|
||||||
|
super(errorCode + ":" + errorDesc);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CanalException(String errorCode, String errorDesc, Throwable cause){
|
||||||
|
super(errorCode + ":" + errorDesc, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CanalException(Throwable cause){
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Throwable fillInStackTrace() {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package com.alibaba.otter.canal.common;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author jianghang 2012-7-12 上午09:39:33
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public interface CanalLifeCycle {
|
||||||
|
|
||||||
|
public void start();
|
||||||
|
|
||||||
|
public void stop();
|
||||||
|
|
||||||
|
public boolean isStart();
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package com.alibaba.otter.canal.common.alarm;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.CanalLifeCycle;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* canal报警处理机制
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-8-22 下午10:08:56
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public interface CanalAlarmHandler extends CanalLifeCycle {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 发送对应destination的报警
|
||||||
|
*
|
||||||
|
* @param destination
|
||||||
|
* @param title
|
||||||
|
* @param msg
|
||||||
|
*/
|
||||||
|
public void sendAlarm(String destination, String msg);
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package com.alibaba.otter.canal.common.alarm;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 基于log的alarm机制实现
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-8-22 下午10:12:35
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class LogAlarmHandler extends AbstractCanalLifeCycle implements CanalAlarmHandler {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(LogAlarmHandler.class);
|
||||||
|
|
||||||
|
public void sendAlarm(String destination, String msg) {
|
||||||
|
logger.error("destination:{}[{}]", new Object[] { destination, msg });
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package com.alibaba.otter.canal.common.utils;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.net.NetworkInterface;
|
||||||
|
import java.net.ServerSocket;
|
||||||
|
import java.util.Enumeration;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class AddressUtils {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(AddressUtils.class);
|
||||||
|
private static final String LOCALHOST_IP = "127.0.0.1";
|
||||||
|
private static final String EMPTY_IP = "0.0.0.0";
|
||||||
|
private static final Pattern IP_PATTERN = Pattern.compile("[0-9]{1,3}(\\.[0-9]{1,3}){3,}");
|
||||||
|
|
||||||
|
public static boolean isAvailablePort(int port) {
|
||||||
|
ServerSocket ss = null;
|
||||||
|
try {
|
||||||
|
ss = new ServerSocket(port);
|
||||||
|
ss.bind(null);
|
||||||
|
return true;
|
||||||
|
} catch (IOException e) {
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
if (ss != null) {
|
||||||
|
try {
|
||||||
|
ss.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isValidHostAddress(InetAddress address) {
|
||||||
|
if (address == null || address.isLoopbackAddress()) return false;
|
||||||
|
String name = address.getHostAddress();
|
||||||
|
return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getHostIp() {
|
||||||
|
InetAddress address = getHostAddress();
|
||||||
|
return address == null ? null : address.getHostAddress();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getHostName() {
|
||||||
|
InetAddress address = getHostAddress();
|
||||||
|
return address == null ? null : address.getHostName();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static InetAddress getHostAddress() {
|
||||||
|
InetAddress localAddress = null;
|
||||||
|
try {
|
||||||
|
localAddress = InetAddress.getLocalHost();
|
||||||
|
if (isValidHostAddress(localAddress)) {
|
||||||
|
return localAddress;
|
||||||
|
}
|
||||||
|
} catch (Throwable e) {
|
||||||
|
logger.warn("Failed to retriving local host ip address, try scan network card ip address. cause: "
|
||||||
|
+ e.getMessage());
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
|
||||||
|
if (interfaces != null) {
|
||||||
|
while (interfaces.hasMoreElements()) {
|
||||||
|
try {
|
||||||
|
NetworkInterface network = interfaces.nextElement();
|
||||||
|
Enumeration<InetAddress> addresses = network.getInetAddresses();
|
||||||
|
if (addresses != null) {
|
||||||
|
while (addresses.hasMoreElements()) {
|
||||||
|
try {
|
||||||
|
InetAddress address = addresses.nextElement();
|
||||||
|
if (isValidHostAddress(address)) {
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
} catch (Throwable e) {
|
||||||
|
logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Throwable e) {
|
||||||
|
logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Throwable e) {
|
||||||
|
logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage());
|
||||||
|
}
|
||||||
|
logger.error("Could not get local host ip address, will use 127.0.0.1 instead.");
|
||||||
|
return localAddress;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
package com.alibaba.otter.canal.common.utils;
|
||||||
|
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.TimeoutException;
|
||||||
|
import java.util.concurrent.locks.AbstractQueuedSynchronizer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 实现一个互斥实现，基于Cocurrent中的{@linkplain AbstractQueuedSynchronizer}实现了自己的sync <br/>
|
||||||
|
* 应用场景：系统初始化/授权控制，没权限时阻塞等待。有权限时所有线程都可以快速通过
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* false : 代表需要被阻塞挂起，等待mutex变为true被唤醒
|
||||||
|
* true : 唤醒被阻塞在false状态下的thread
|
||||||
|
*
|
||||||
|
* BooleanMutex mutex = new BooleanMutex(true);
|
||||||
|
* try {
|
||||||
|
* mutex.get(); //当前状态为true, 不会被阻塞
|
||||||
|
* } catch (InterruptedException e) {
|
||||||
|
* // do something
|
||||||
|
* }
|
||||||
|
*
|
||||||
|
* mutex.set(false);
|
||||||
|
* try {
|
||||||
|
* mutex.get(); //当前状态为false, 会被阻塞直到另一个线程调用mutex.set(true);
|
||||||
|
* } catch (InterruptedException e) {
|
||||||
|
* // do something
|
||||||
|
* }
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author jianghang 2011-9-23 上午09:58:03
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class BooleanMutex {
|
||||||
|
|
||||||
|
private Sync sync;
|
||||||
|
|
||||||
|
public BooleanMutex(){
|
||||||
|
sync = new Sync();
|
||||||
|
set(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public BooleanMutex(Boolean mutex){
|
||||||
|
sync = new Sync();
|
||||||
|
set(mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 阻塞等待Boolean为true
|
||||||
|
*
|
||||||
|
* @throws InterruptedException
|
||||||
|
*/
|
||||||
|
public void get() throws InterruptedException {
|
||||||
|
sync.innerGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 阻塞等待Boolean为true,允许设置超时时间
|
||||||
|
*
|
||||||
|
* @param timeout
|
||||||
|
* @param unit
|
||||||
|
* @throws InterruptedException
|
||||||
|
* @throws TimeoutException
|
||||||
|
*/
|
||||||
|
public void get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
|
||||||
|
sync.innerGet(unit.toNanos(timeout));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 重新设置对应的Boolean mutex
|
||||||
|
*
|
||||||
|
* @param mutex
|
||||||
|
*/
|
||||||
|
public void set(Boolean mutex) {
|
||||||
|
if (mutex) {
|
||||||
|
sync.innerSetTrue();
|
||||||
|
} else {
|
||||||
|
sync.innerSetFalse();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean state() {
|
||||||
|
return sync.innerState();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synchronization control for BooleanMutex. Uses AQS sync state to
|
||||||
|
* represent run status
|
||||||
|
*/
|
||||||
|
private final class Sync extends AbstractQueuedSynchronizer {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 2559471934544126329L;
|
||||||
|
/** State value representing that TRUE */
|
||||||
|
private static final int TRUE = 1;
|
||||||
|
/** State value representing that FALSE */
|
||||||
|
private static final int FALSE = 2;
|
||||||
|
|
||||||
|
private boolean isTrue(int state) {
|
||||||
|
return (state & TRUE) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 实现AQS的接口，获取共享锁的判断
|
||||||
|
*/
|
||||||
|
protected int tryAcquireShared(int state) {
|
||||||
|
// 如果为true，直接允许获取锁对象
|
||||||
|
// 如果为false，进入阻塞队列，等待被唤醒
|
||||||
|
return isTrue(getState()) ? 1 : -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 实现AQS的接口，释放共享锁的判断
|
||||||
|
*/
|
||||||
|
protected boolean tryReleaseShared(int ignore) {
|
||||||
|
// 始终返回true，代表可以release
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
boolean innerState() {
|
||||||
|
return isTrue(getState());
|
||||||
|
}
|
||||||
|
|
||||||
|
void innerGet() throws InterruptedException {
|
||||||
|
acquireSharedInterruptibly(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void innerGet(long nanosTimeout) throws InterruptedException, TimeoutException {
|
||||||
|
if (!tryAcquireSharedNanos(0, nanosTimeout)) throw new TimeoutException();
|
||||||
|
}
|
||||||
|
|
||||||
|
void innerSetTrue() {
|
||||||
|
for (;;) {
|
||||||
|
int s = getState();
|
||||||
|
if (s == TRUE) {
|
||||||
|
return; // 直接退出
|
||||||
|
}
|
||||||
|
if (compareAndSetState(s, TRUE)) {// cas更新状态，避免并发更新true操作
|
||||||
|
releaseShared(0);// 释放一下锁对象，唤醒一下阻塞的Thread
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void innerSetFalse() {
|
||||||
|
for (;;) {
|
||||||
|
int s = getState();
|
||||||
|
if (s == FALSE) {
|
||||||
|
return; // 直接退出
|
||||||
|
}
|
||||||
|
if (compareAndSetState(s, FALSE)) {// cas更新状态，避免并发更新false操作
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package com.alibaba.otter.canal.common.utils;
|
||||||
|
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.builder.ToStringStyle;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Otter项目内部使用的ToStringStyle
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* 默认Style输出格式：
|
||||||
|
* Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author jianghang 2010-6-18 上午11:35:27
|
||||||
|
*/
|
||||||
|
public class CanalToStringStyle extends ToStringStyle {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = -6568177374288222145L;
|
||||||
|
|
||||||
|
private static final String DEFAULT_TIME = "yyyy-MM-dd HH:mm:ss";
|
||||||
|
private static final String DEFAULT_DAY = "yyyy-MM-dd";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
* 输出格式：
|
||||||
|
* Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
|
||||||
|
* </pre>
|
||||||
|
*/
|
||||||
|
public static final ToStringStyle TIME_STYLE = new OtterDateStyle(DEFAULT_TIME);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
* 输出格式：
|
||||||
|
* Person[name=John Doe,age=33,smoker=false ,day=2010-04-01]
|
||||||
|
* </pre>
|
||||||
|
*/
|
||||||
|
public static final ToStringStyle DAY_STYLE = new OtterDateStyle(DEFAULT_DAY);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
* 输出格式：
|
||||||
|
* Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
|
||||||
|
* </pre>
|
||||||
|
*/
|
||||||
|
public static final ToStringStyle DEFAULT_STYLE = CanalToStringStyle.TIME_STYLE;
|
||||||
|
|
||||||
|
// =========================== 自定义style =============================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 支持日期格式化的ToStringStyle
|
||||||
|
*
|
||||||
|
* @author li.jinl
|
||||||
|
*/
|
||||||
|
private static class OtterDateStyle extends ToStringStyle {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 5208917932254652886L;
|
||||||
|
|
||||||
|
// 日期format格式
|
||||||
|
private String pattern;
|
||||||
|
|
||||||
|
public OtterDateStyle(String pattern){
|
||||||
|
super();
|
||||||
|
this.setUseShortClassName(true);
|
||||||
|
this.setUseIdentityHashCode(false);
|
||||||
|
// 设置日期format格式
|
||||||
|
this.pattern = pattern;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void appendDetail(StringBuffer buffer, String fieldName, Object value) {
|
||||||
|
// 增加自定义的date对象处理
|
||||||
|
if (value instanceof Date) {
|
||||||
|
value = new SimpleDateFormat(pattern).format(value);
|
||||||
|
}
|
||||||
|
// 后续可以增加其他自定义对象处理
|
||||||
|
buffer.append(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package com.alibaba.otter.canal.common.utils;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSON;
|
||||||
|
import com.alibaba.fastjson.TypeReference;
|
||||||
|
import com.alibaba.fastjson.serializer.JSONSerializer;
|
||||||
|
import com.alibaba.fastjson.serializer.PropertyFilter;
|
||||||
|
import com.alibaba.fastjson.serializer.SerializeWriter;
|
||||||
|
import com.alibaba.fastjson.serializer.SerializerFeature;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 字节处理相关工具类
|
||||||
|
*
|
||||||
|
* @author jianghang
|
||||||
|
*/
|
||||||
|
public class JsonUtils {
|
||||||
|
|
||||||
|
public static <T> T unmarshalFromByte(byte[] bytes, Class<T> targetClass) {
|
||||||
|
return (T) JSON.parseObject(bytes, targetClass);// 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T unmarshalFromByte(byte[] bytes, TypeReference<T> type) {
|
||||||
|
return (T) JSON.parseObject(bytes, type.getType());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] marshalToByte(Object obj) {
|
||||||
|
return JSON.toJSONBytes(obj); // 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] marshalToByte(Object obj, SerializerFeature... features) {
|
||||||
|
return JSON.toJSONBytes(obj, features); // 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T unmarshalFromString(String json, Class<T> targetClass) {
|
||||||
|
return (T) JSON.parseObject(json, targetClass);// 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T unmarshalFromString(String json, TypeReference<T> type) {
|
||||||
|
return (T) JSON.parseObject(json, type);// 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String marshalToString(Object obj) {
|
||||||
|
return JSON.toJSONString(obj); // 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String marshalToString(Object obj, SerializerFeature... features) {
|
||||||
|
return JSON.toJSONString(obj, features); // 默认为UTF-8
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 可以允许指定一些过滤字段进行生成json对象
|
||||||
|
*/
|
||||||
|
public static String marshalToString(Object obj, String... fliterFields) {
|
||||||
|
final List<String> propertyFliters = Arrays.asList(fliterFields);
|
||||||
|
SerializeWriter out = new SerializeWriter();
|
||||||
|
try {
|
||||||
|
JSONSerializer serializer = new JSONSerializer(out);
|
||||||
|
serializer.getPropertyFilters().add(new PropertyFilter() {
|
||||||
|
|
||||||
|
public boolean apply(Object source, String name, Object value) {
|
||||||
|
return !propertyFliters.contains(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
serializer.write(obj);
|
||||||
|
return out.toString();
|
||||||
|
} finally {
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package com.alibaba.otter.canal.common.utils;
|
||||||
|
|
||||||
|
import java.lang.Thread.UncaughtExceptionHandler;
|
||||||
|
import java.util.concurrent.ThreadFactory;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author zebin.xuzb 2012-9-20 下午3:47:47
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class NamedThreadFactory implements ThreadFactory {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(NamedThreadFactory.class);
|
||||||
|
final private static String DEFAULT_NAME = "canal-worker";
|
||||||
|
final private String name;
|
||||||
|
final private boolean daemon;
|
||||||
|
final private ThreadGroup group;
|
||||||
|
final private AtomicInteger threadNumber = new AtomicInteger(0);
|
||||||
|
final static UncaughtExceptionHandler uncaughtExceptionHandler = new UncaughtExceptionHandler() {
|
||||||
|
|
||||||
|
public void uncaughtException(Thread t,
|
||||||
|
Throwable e) {
|
||||||
|
logger.error("from " + t.getName(), e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
public NamedThreadFactory(){
|
||||||
|
this(DEFAULT_NAME, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NamedThreadFactory(String name){
|
||||||
|
this(name, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public NamedThreadFactory(String name, boolean daemon){
|
||||||
|
this.name = name;
|
||||||
|
this.daemon = daemon;
|
||||||
|
SecurityManager s = System.getSecurityManager();
|
||||||
|
group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Thread newThread(Runnable r) {
|
||||||
|
Thread t = new Thread(group, r, name + "-" + threadNumber.getAndIncrement(), 0);
|
||||||
|
t.setDaemon(daemon);
|
||||||
|
if (t.getPriority() != Thread.NORM_PRIORITY) {
|
||||||
|
t.setPriority(Thread.NORM_PRIORITY);
|
||||||
|
}
|
||||||
|
|
||||||
|
t.setUncaughtExceptionHandler(uncaughtExceptionHandler);
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package com.alibaba.otter.canal.common.utils;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.net.URLDecoder;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Scanner;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author zebin.xuzb 2012-11-15 下午3:53:09
|
||||||
|
* @since 1.0.0
|
||||||
|
*/
|
||||||
|
public final class UriUtils {
|
||||||
|
|
||||||
|
private final static String SPLIT = "&";
|
||||||
|
private final static String EQUAL = "=";
|
||||||
|
private final static String DEFAULT_ENCODING = "ISO_8859_1";
|
||||||
|
|
||||||
|
private UriUtils(){
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<String, String> parseQuery(final String uriString) {
|
||||||
|
URI uri = null;
|
||||||
|
try {
|
||||||
|
uri = new URI(uriString);
|
||||||
|
} catch (URISyntaxException e) {
|
||||||
|
throw new IllegalArgumentException(e);
|
||||||
|
}
|
||||||
|
return parseQuery(uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<String, String> parseQuery(final String uriString, final String encoding) {
|
||||||
|
URI uri = null;
|
||||||
|
try {
|
||||||
|
uri = new URI(uriString);
|
||||||
|
} catch (URISyntaxException e) {
|
||||||
|
throw new IllegalArgumentException(e);
|
||||||
|
}
|
||||||
|
return parseQuery(uri, encoding);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<String, String> parseQuery(final URI uri) {
|
||||||
|
return parseQuery(uri, DEFAULT_ENCODING);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<String, String> parseQuery(final URI uri, final String encoding) {
|
||||||
|
if (uri == null || StringUtils.isBlank(uri.getQuery())) {
|
||||||
|
return Collections.EMPTY_MAP;
|
||||||
|
}
|
||||||
|
String query = uri.getRawQuery();
|
||||||
|
HashMap<String, String> params = new HashMap<String, String>();
|
||||||
|
Scanner scan = new Scanner(query);
|
||||||
|
scan.useDelimiter(SPLIT);
|
||||||
|
while (scan.hasNext()) {
|
||||||
|
String token = scan.next().trim();
|
||||||
|
String[] pair = token.split(EQUAL);
|
||||||
|
String key = decode(pair[0], encoding);
|
||||||
|
String value = null;
|
||||||
|
if (pair.length == 2) {
|
||||||
|
value = decode(pair[1], encoding);
|
||||||
|
}
|
||||||
|
params.put(key, value);
|
||||||
|
}
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String decode(final String content, final String encoding) {
|
||||||
|
try {
|
||||||
|
return URLDecoder.decode(content, encoding != null ? encoding : DEFAULT_ENCODING);
|
||||||
|
} catch (UnsupportedEncodingException e) {
|
||||||
|
throw new IllegalArgumentException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.exception.ZkMarshallingError;
|
||||||
|
import org.I0Itec.zkclient.serialize.ZkSerializer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 基于string的序列化方式
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-7-11 下午02:57:09
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ByteSerializer implements ZkSerializer {
|
||||||
|
|
||||||
|
public Object deserialize(final byte[] bytes) throws ZkMarshallingError {
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public byte[] serialize(final Object data) throws ZkMarshallingError {
|
||||||
|
try {
|
||||||
|
if (data instanceof byte[]) {
|
||||||
|
return (byte[]) data;
|
||||||
|
} else {
|
||||||
|
return ((String) data).getBytes("utf-8");
|
||||||
|
}
|
||||||
|
} catch (final UnsupportedEncodingException e) {
|
||||||
|
throw new ZkMarshallingError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.exception.ZkMarshallingError;
|
||||||
|
import org.I0Itec.zkclient.serialize.ZkSerializer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 基于string的序列化方式
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-7-11 下午02:57:09
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class StringSerializer implements ZkSerializer {
|
||||||
|
|
||||||
|
public Object deserialize(final byte[] bytes) throws ZkMarshallingError {
|
||||||
|
try {
|
||||||
|
return new String(bytes, "utf-8");
|
||||||
|
} catch (final UnsupportedEncodingException e) {
|
||||||
|
throw new ZkMarshallingError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public byte[] serialize(final Object data) throws ZkMarshallingError {
|
||||||
|
try {
|
||||||
|
return ((String) data).getBytes("utf-8");
|
||||||
|
} catch (final UnsupportedEncodingException e) {
|
||||||
|
throw new ZkMarshallingError(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.IZkConnection;
|
||||||
|
import org.I0Itec.zkclient.ZkClient;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkInterruptedException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNoNodeException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
|
||||||
|
import org.I0Itec.zkclient.serialize.ZkSerializer;
|
||||||
|
import org.apache.zookeeper.CreateMode;
|
||||||
|
|
||||||
|
import com.google.common.base.Function;
|
||||||
|
import com.google.common.collect.MapMaker;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用自定义的ZooKeeperx for zk connection
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-7-10 下午02:31:15
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ZkClientx extends ZkClient {
|
||||||
|
|
||||||
|
// 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
|
||||||
|
private static Map<String, ZkClientx> clients = new MapMaker().makeComputingMap(new Function<String, ZkClientx>() {
|
||||||
|
|
||||||
|
public ZkClientx apply(String servers) {
|
||||||
|
return new ZkClientx(servers);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
public static ZkClientx getZkClient(String servers) {
|
||||||
|
return clients.get(servers);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZkClientx(String serverstring){
|
||||||
|
this(serverstring, Integer.MAX_VALUE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZkClientx(String zkServers, int connectionTimeout){
|
||||||
|
this(new ZooKeeperx(zkServers), connectionTimeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZkClientx(String zkServers, int sessionTimeout, int connectionTimeout){
|
||||||
|
this(new ZooKeeperx(zkServers, sessionTimeout), connectionTimeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZkClientx(String zkServers, int sessionTimeout, int connectionTimeout, ZkSerializer zkSerializer){
|
||||||
|
this(new ZooKeeperx(zkServers, sessionTimeout), connectionTimeout, zkSerializer);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ZkClientx(IZkConnection connection, int connectionTimeout){
|
||||||
|
this(connection, connectionTimeout, new ByteSerializer());
|
||||||
|
}
|
||||||
|
|
||||||
|
private ZkClientx(IZkConnection zkConnection, int connectionTimeout, ZkSerializer zkSerializer){
|
||||||
|
super(zkConnection, connectionTimeout, zkSerializer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a persistent Sequential node.
|
||||||
|
*
|
||||||
|
* @param path
|
||||||
|
* @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
|
||||||
|
* in case the path already exists
|
||||||
|
* @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
|
||||||
|
* @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
|
||||||
|
* @throws ZkException if any ZooKeeper exception occurred
|
||||||
|
* @throws RuntimeException if any other exception occurs
|
||||||
|
*/
|
||||||
|
public String createPersistentSequential(String path, boolean createParents) throws ZkInterruptedException,
|
||||||
|
IllegalArgumentException, ZkException,
|
||||||
|
RuntimeException {
|
||||||
|
try {
|
||||||
|
return create(path, null, CreateMode.PERSISTENT_SEQUENTIAL);
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
if (!createParents) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
String parentDir = path.substring(0, path.lastIndexOf('/'));
|
||||||
|
createPersistent(parentDir, createParents);
|
||||||
|
return createPersistentSequential(path, createParents);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a persistent Sequential node.
|
||||||
|
*
|
||||||
|
* @param path
|
||||||
|
* @param data
|
||||||
|
* @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
|
||||||
|
* in case the path already exists
|
||||||
|
* @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
|
||||||
|
* @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
|
||||||
|
* @throws ZkException if any ZooKeeper exception occurred
|
||||||
|
* @throws RuntimeException if any other exception occurs
|
||||||
|
*/
|
||||||
|
public String createPersistentSequential(String path, Object data, boolean createParents)
|
||||||
|
throws ZkInterruptedException,
|
||||||
|
IllegalArgumentException,
|
||||||
|
ZkException,
|
||||||
|
RuntimeException {
|
||||||
|
try {
|
||||||
|
return create(path, data, CreateMode.PERSISTENT_SEQUENTIAL);
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
if (!createParents) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
String parentDir = path.substring(0, path.lastIndexOf('/'));
|
||||||
|
createPersistent(parentDir, createParents);
|
||||||
|
return createPersistentSequential(path, data, createParents);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a persistent Sequential node.
|
||||||
|
*
|
||||||
|
* @param path
|
||||||
|
* @param data
|
||||||
|
* @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
|
||||||
|
* in case the path already exists
|
||||||
|
* @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
|
||||||
|
* @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
|
||||||
|
* @throws ZkException if any ZooKeeper exception occurred
|
||||||
|
* @throws RuntimeException if any other exception occurs
|
||||||
|
*/
|
||||||
|
public void createPersistent(String path, Object data, boolean createParents) throws ZkInterruptedException,
|
||||||
|
IllegalArgumentException, ZkException,
|
||||||
|
RuntimeException {
|
||||||
|
try {
|
||||||
|
create(path, data, CreateMode.PERSISTENT);
|
||||||
|
} catch (ZkNodeExistsException e) {
|
||||||
|
if (!createParents) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
if (!createParents) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
String parentDir = path.substring(0, path.lastIndexOf('/'));
|
||||||
|
createPersistent(parentDir, createParents);
|
||||||
|
createPersistent(path, data, createParents);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,181 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.concurrent.locks.Lock;
|
||||||
|
import java.util.concurrent.locks.ReentrantLock;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.IZkConnection;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkException;
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.apache.zookeeper.ClientCnxn;
|
||||||
|
import org.apache.zookeeper.CreateMode;
|
||||||
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
import org.apache.zookeeper.Watcher;
|
||||||
|
import org.apache.zookeeper.ZooKeeper;
|
||||||
|
import org.apache.zookeeper.ZooDefs.Ids;
|
||||||
|
import org.apache.zookeeper.ZooKeeper.States;
|
||||||
|
import org.apache.zookeeper.client.ConnectStringParser;
|
||||||
|
import org.apache.zookeeper.client.HostProvider;
|
||||||
|
import org.apache.zookeeper.client.StaticHostProvider;
|
||||||
|
import org.apache.zookeeper.data.Stat;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.util.ReflectionUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 封装了ZooKeeper，使其支持节点的优先顺序，比如美国机房的节点会优先加载美国对应的zk集群列表，都失败后才会选择加载杭州的zk集群列表 *
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-7-10 下午02:31:42
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ZooKeeperx implements IZkConnection {
|
||||||
|
|
||||||
|
private static final String SERVER_COMMA = ";";
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ZooKeeperx.class);
|
||||||
|
private static final Field clientCnxnField = ReflectionUtils.findField(ZooKeeper.class, "cnxn");
|
||||||
|
private static final Field hostProviderField = ReflectionUtils.findField(ClientCnxn.class, "hostProvider");
|
||||||
|
private static final Field serverAddressesField = ReflectionUtils.findField(StaticHostProvider.class,
|
||||||
|
"serverAddresses");
|
||||||
|
private static final int DEFAULT_SESSION_TIMEOUT = 90000;
|
||||||
|
|
||||||
|
private ZooKeeper _zk = null;
|
||||||
|
private Lock _zookeeperLock = new ReentrantLock();
|
||||||
|
|
||||||
|
private final List<String> _servers;
|
||||||
|
private final int _sessionTimeOut;
|
||||||
|
|
||||||
|
public ZooKeeperx(String zkServers){
|
||||||
|
this(zkServers, DEFAULT_SESSION_TIMEOUT);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZooKeeperx(String zkServers, int sessionTimeOut){
|
||||||
|
_servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA));
|
||||||
|
_sessionTimeOut = sessionTimeOut;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void connect(Watcher watcher) {
|
||||||
|
_zookeeperLock.lock();
|
||||||
|
try {
|
||||||
|
if (_zk != null) {
|
||||||
|
throw new IllegalStateException("zk client has already been started");
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
logger.debug("Creating new ZookKeeper instance to connect to " + _servers + ".");
|
||||||
|
_zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher);
|
||||||
|
configMutliCluster(_zk);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new ZkException("Unable to connect to " + _servers, e);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
_zookeeperLock.unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void close() throws InterruptedException {
|
||||||
|
_zookeeperLock.lock();
|
||||||
|
try {
|
||||||
|
if (_zk != null) {
|
||||||
|
logger.debug("Closing ZooKeeper connected to " + _servers);
|
||||||
|
_zk.close();
|
||||||
|
_zk = null;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
_zookeeperLock.unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException {
|
||||||
|
return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void delete(String path) throws InterruptedException, KeeperException {
|
||||||
|
_zk.delete(path, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException {
|
||||||
|
return _zk.exists(path, watch) != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<String> getChildren(final String path, final boolean watch) throws KeeperException,
|
||||||
|
InterruptedException {
|
||||||
|
return _zk.getChildren(path, watch);
|
||||||
|
}
|
||||||
|
|
||||||
|
public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException {
|
||||||
|
return _zk.getData(path, watch, stat);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writeData(String path, byte[] data) throws KeeperException, InterruptedException {
|
||||||
|
writeData(path, data, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException {
|
||||||
|
_zk.setData(path, data, version);
|
||||||
|
}
|
||||||
|
|
||||||
|
public States getZookeeperState() {
|
||||||
|
return _zk != null ? _zk.getState() : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ZooKeeper getZookeeper() {
|
||||||
|
return _zk;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getCreateTime(String path) throws KeeperException, InterruptedException {
|
||||||
|
Stat stat = _zk.exists(path, false);
|
||||||
|
if (stat != null) {
|
||||||
|
return stat.getCtime();
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getServers() {
|
||||||
|
return StringUtils.join(_servers, SERVER_COMMA);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===============================
|
||||||
|
|
||||||
|
public void configMutliCluster(ZooKeeper zk) {
|
||||||
|
if (_servers.size() == 1) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
String cluster1 = _servers.get(0);
|
||||||
|
try {
|
||||||
|
if (_servers.size() > 1) {
|
||||||
|
// 强制的声明accessible
|
||||||
|
ReflectionUtils.makeAccessible(clientCnxnField);
|
||||||
|
ReflectionUtils.makeAccessible(hostProviderField);
|
||||||
|
ReflectionUtils.makeAccessible(serverAddressesField);
|
||||||
|
|
||||||
|
// 添加第二组集群列表
|
||||||
|
for (int i = 1; i < _servers.size(); i++) {
|
||||||
|
String cluster = _servers.get(i);
|
||||||
|
// 强制获取zk中的地址信息
|
||||||
|
ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
|
||||||
|
HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
|
||||||
|
List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(
|
||||||
|
serverAddressesField,
|
||||||
|
hostProvider);
|
||||||
|
// 添加第二组集群列表
|
||||||
|
serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
try {
|
||||||
|
if (zk != null) {
|
||||||
|
zk.close();
|
||||||
|
}
|
||||||
|
} catch (InterruptedException ie) {
|
||||||
|
// ignore interrupt
|
||||||
|
}
|
||||||
|
throw new ZkException("zookeeper_create_error, serveraddrs=" + cluster1, e);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
+181
@@ -0,0 +1,181 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper;
|
||||||
|
|
||||||
|
import java.text.MessageFormat;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 存储结构：
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* /otter
|
||||||
|
* canal
|
||||||
|
* cluster
|
||||||
|
* destinations
|
||||||
|
* dest1
|
||||||
|
* running (EPHEMERAL)
|
||||||
|
* cluster
|
||||||
|
* client1
|
||||||
|
* running (EPHEMERAL)
|
||||||
|
* cluster
|
||||||
|
* filter
|
||||||
|
* cursor
|
||||||
|
* mark
|
||||||
|
* 1
|
||||||
|
* 2
|
||||||
|
* 3
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author zebin.xuzb @ 2012-6-21
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ZookeeperPathUtils {
|
||||||
|
|
||||||
|
public static final String ZOOKEEPER_SEPARATOR = "/";
|
||||||
|
|
||||||
|
public static final String OTTER_ROOT_NODE = ZOOKEEPER_SEPARATOR + "otter";
|
||||||
|
|
||||||
|
public static final String CANAL_ROOT_NODE = OTTER_ROOT_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ "canal";
|
||||||
|
|
||||||
|
public static final String DESTINATION_ROOT_NODE = CANAL_ROOT_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ "destinations";
|
||||||
|
|
||||||
|
public static final String FILTER_NODE = "filter";
|
||||||
|
|
||||||
|
public static final String BATCH_MARK_NODE = "mark";
|
||||||
|
|
||||||
|
public static final String PARSE_NODE = "parse";
|
||||||
|
|
||||||
|
public static final String CURSOR_NODE = "cursor";
|
||||||
|
|
||||||
|
public static final String RUNNING_NODE = "running";
|
||||||
|
|
||||||
|
public static final String CLUSTER_NODE = "cluster";
|
||||||
|
|
||||||
|
public static final String DESTINATION_NODE = DESTINATION_ROOT_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + "{0}";
|
||||||
|
|
||||||
|
public static final String DESTINATION_PARSE_NODE = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ PARSE_NODE;
|
||||||
|
|
||||||
|
public static final String DESTINATION_CLIENTID_NODE = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ "{1}";
|
||||||
|
|
||||||
|
public static final String DESTINATION_CURSOR_NODE = DESTINATION_CLIENTID_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + CURSOR_NODE;
|
||||||
|
|
||||||
|
public static final String DESTINATION_CLIENTID_FILTER_NODE = DESTINATION_CLIENTID_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + FILTER_NODE;
|
||||||
|
|
||||||
|
public static final String DESTINATION_CLIENTID_BATCH_MARK_NODE = DESTINATION_CLIENTID_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + BATCH_MARK_NODE;
|
||||||
|
|
||||||
|
public static final String DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH = DESTINATION_CLIENTID_BATCH_MARK_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + "{2}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 服务端当前正在提供服务的running节点
|
||||||
|
*/
|
||||||
|
public static final String DESTINATION_RUNNING_NODE = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ RUNNING_NODE;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端当前正在工作的running节点
|
||||||
|
*/
|
||||||
|
public static final String DESTINATION_CLIENTID_RUNNING_NODE = DESTINATION_CLIENTID_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + RUNNING_NODE;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 整个canal server的集群列表
|
||||||
|
*/
|
||||||
|
public static final String CANAL_CLUSTER_ROOT_NODE = CANAL_ROOT_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ CLUSTER_NODE;
|
||||||
|
|
||||||
|
public static final String CANAL_CLUSTER_NODE = CANAL_CLUSTER_ROOT_NODE
|
||||||
|
+ ZOOKEEPER_SEPARATOR + "{0}";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 针对某个destination的工作的集群列表
|
||||||
|
*/
|
||||||
|
public static final String DESTINATION_CLUSTER_ROOT = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
|
||||||
|
+ CLUSTER_NODE;
|
||||||
|
public static final String DESTINATION_CLUSTER_NODE = DESTINATION_CLUSTER_ROOT
|
||||||
|
+ ZOOKEEPER_SEPARATOR + "{1}";
|
||||||
|
|
||||||
|
public static String getDestinationPath(String destinationName) {
|
||||||
|
return MessageFormat.format(DESTINATION_NODE, destinationName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getClientIdNodePath(String destinationName, short clientId) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLIENTID_NODE, destinationName, String.valueOf(clientId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getFilterPath(String destinationName, short clientId) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLIENTID_FILTER_NODE, destinationName, String.valueOf(clientId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getBatchMarkPath(String destinationName, short clientId) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_NODE, destinationName, String.valueOf(clientId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName,
|
||||||
|
String.valueOf(clientId), getBatchMarkNode(batchId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getCursorPath(String destination, short clientId) {
|
||||||
|
return MessageFormat.format(DESTINATION_CURSOR_NODE, destination, String.valueOf(clientId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getCanalClusterNode(String node) {
|
||||||
|
return MessageFormat.format(CANAL_CLUSTER_NODE, node);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 服务端当前正在提供服务的running节点
|
||||||
|
*/
|
||||||
|
public static String getDestinationServerRunning(String destination) {
|
||||||
|
return MessageFormat.format(DESTINATION_RUNNING_NODE, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 客户端当前正在工作的running节点
|
||||||
|
*/
|
||||||
|
public static String getDestinationClientRunning(String destination, short clientId) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLIENTID_RUNNING_NODE, destination, String.valueOf(clientId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getDestinationClusterNode(String destination, String node) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLUSTER_NODE, destination, node);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getDestinationClusterRoot(String destination) {
|
||||||
|
return MessageFormat.format(DESTINATION_CLUSTER_ROOT, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getParsePath(String destination) {
|
||||||
|
return MessageFormat.format(DESTINATION_PARSE_NODE, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将batchNode转换为Long
|
||||||
|
*/
|
||||||
|
public static short getClientId(String clientNode) {
|
||||||
|
return Short.valueOf(clientNode);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将batchNode转换为Long
|
||||||
|
*/
|
||||||
|
public static long getBatchMarkId(String batchMarkNode) {
|
||||||
|
return Long.valueOf(batchMarkNode);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将batchId转化为zookeeper中的node名称
|
||||||
|
*/
|
||||||
|
public static String getBatchMarkNode(Long batchId) {
|
||||||
|
return StringUtils.leftPad(String.valueOf(batchId.intValue()), 10, '0');
|
||||||
|
}
|
||||||
|
}
|
||||||
+59
@@ -0,0 +1,59 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper.running;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.builder.ToStringBuilder;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 服务端running状态信息
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-11-22 下午03:11:30
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ServerRunningData implements Serializable {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 92260481691855281L;
|
||||||
|
|
||||||
|
private Long cid;
|
||||||
|
private String address;
|
||||||
|
private boolean active = true;
|
||||||
|
|
||||||
|
public ServerRunningData(){
|
||||||
|
}
|
||||||
|
|
||||||
|
public ServerRunningData(Long cid, String address){
|
||||||
|
this.cid = cid;
|
||||||
|
this.address = address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getCid() {
|
||||||
|
return cid;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCid(Long cid) {
|
||||||
|
this.cid = cid;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAddress() {
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAddress(String address) {
|
||||||
|
this.address = address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isActive() {
|
||||||
|
return active;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setActive(boolean active) {
|
||||||
|
this.active = active;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+31
@@ -0,0 +1,31 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper.running;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 触发一下mainstem发生切换
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-9-11 下午02:26:03
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public interface ServerRunningListener {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 启动时回调做点事情
|
||||||
|
*/
|
||||||
|
public void processStart();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 关闭时回调做点事情
|
||||||
|
*/
|
||||||
|
public void processStop();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 触发现在轮到自己做为active，需要载入上一个active的上下文数据
|
||||||
|
*/
|
||||||
|
public void processActiveEnter();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 触发一下当前active模式失败
|
||||||
|
*/
|
||||||
|
public void processActiveExit();
|
||||||
|
|
||||||
|
}
|
||||||
+273
@@ -0,0 +1,273 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper.running;
|
||||||
|
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.IZkDataListener;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkInterruptedException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNoNodeException;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNodeExistsException;
|
||||||
|
import org.apache.zookeeper.CreateMode;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.slf4j.MDC;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
|
||||||
|
import com.alibaba.otter.canal.common.utils.BooleanMutex;
|
||||||
|
import com.alibaba.otter.canal.common.utils.JsonUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 针对server的running节点控制
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-11-22 下午02:59:42
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ServerRunningMonitor extends AbstractCanalLifeCycle {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(ServerRunningMonitor.class);
|
||||||
|
private ZkClientx zkClient;
|
||||||
|
private String destination;
|
||||||
|
private IZkDataListener dataListener;
|
||||||
|
private BooleanMutex mutex = new BooleanMutex(false);
|
||||||
|
private volatile boolean release = false;
|
||||||
|
// 当前服务节点状态信息
|
||||||
|
private ServerRunningData serverData;
|
||||||
|
// 当前实际运行的节点状态信息
|
||||||
|
private volatile ServerRunningData activeData;
|
||||||
|
private ScheduledExecutorService delayExector = Executors.newScheduledThreadPool(1);
|
||||||
|
private int delayTime = 5;
|
||||||
|
private ServerRunningListener listener;
|
||||||
|
|
||||||
|
public ServerRunningMonitor(ServerRunningData serverData){
|
||||||
|
this();
|
||||||
|
this.serverData = serverData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ServerRunningMonitor(){
|
||||||
|
// 创建父节点
|
||||||
|
dataListener = new IZkDataListener() {
|
||||||
|
|
||||||
|
public void handleDataChange(String dataPath, Object data) throws Exception {
|
||||||
|
MDC.put("destination", destination);
|
||||||
|
ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
|
||||||
|
if (!isMine(runningData.getAddress())) {
|
||||||
|
mutex.set(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
|
||||||
|
release = true;
|
||||||
|
releaseRunning();// 彻底释放mainstem
|
||||||
|
}
|
||||||
|
|
||||||
|
activeData = (ServerRunningData) runningData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleDataDeleted(String dataPath) throws Exception {
|
||||||
|
MDC.put("destination", destination);
|
||||||
|
mutex.set(false);
|
||||||
|
if (!release && activeData != null && isMine(activeData.getAddress())) {
|
||||||
|
// 如果上一次active的状态就是本机，则即时触发一下active抢占
|
||||||
|
initRunning();
|
||||||
|
} else {
|
||||||
|
// 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
|
||||||
|
delayExector.schedule(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
initRunning();
|
||||||
|
}
|
||||||
|
}, delayTime, TimeUnit.SECONDS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start() {
|
||||||
|
super.start();
|
||||||
|
processStart();
|
||||||
|
if (zkClient != null) {
|
||||||
|
// 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
|
||||||
|
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||||
|
zkClient.subscribeDataChanges(path, dataListener);
|
||||||
|
|
||||||
|
initRunning();
|
||||||
|
} else {
|
||||||
|
processActiveEnter();// 没有zk，直接启动
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void release() {
|
||||||
|
if (zkClient != null) {
|
||||||
|
releaseRunning(); // 尝试一下release
|
||||||
|
} else {
|
||||||
|
processActiveExit(); // 没有zk，直接启动
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() {
|
||||||
|
super.stop();
|
||||||
|
|
||||||
|
if (zkClient != null) {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||||
|
zkClient.unsubscribeDataChanges(path, dataListener);
|
||||||
|
|
||||||
|
releaseRunning(); // 尝试一下release
|
||||||
|
} else {
|
||||||
|
processActiveExit(); // 没有zk，直接启动
|
||||||
|
}
|
||||||
|
processStop();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initRunning() {
|
||||||
|
if (!isStart()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||||
|
// 序列化
|
||||||
|
byte[] bytes = JsonUtils.marshalToByte(serverData);
|
||||||
|
try {
|
||||||
|
mutex.set(false);
|
||||||
|
zkClient.create(path, bytes, CreateMode.EPHEMERAL);
|
||||||
|
activeData = serverData;
|
||||||
|
processActiveEnter();// 触发一下事件
|
||||||
|
mutex.set(true);
|
||||||
|
} catch (ZkNodeExistsException e) {
|
||||||
|
bytes = zkClient.readData(path, true);
|
||||||
|
if (bytes == null) {// 如果不存在节点，立即尝试一次
|
||||||
|
initRunning();
|
||||||
|
} else {
|
||||||
|
activeData = JsonUtils.unmarshalFromByte(bytes, ServerRunningData.class);
|
||||||
|
}
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
zkClient.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true); // 尝试创建父节点
|
||||||
|
initRunning();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 阻塞等待自己成为active，如果自己成为active，立马返回
|
||||||
|
*
|
||||||
|
* @throws InterruptedException
|
||||||
|
*/
|
||||||
|
public void waitForActive() throws InterruptedException {
|
||||||
|
initRunning();
|
||||||
|
mutex.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 检查当前的状态
|
||||||
|
*/
|
||||||
|
public boolean check() {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||||
|
try {
|
||||||
|
byte[] bytes = zkClient.readData(path);
|
||||||
|
ServerRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ServerRunningData.class);
|
||||||
|
activeData = eventData;// 更新下为最新值
|
||||||
|
// 检查下nid是否为自己
|
||||||
|
boolean result = isMine(activeData.getAddress());
|
||||||
|
if (!result) {
|
||||||
|
logger.warn("canal is running in node[{}] , but not in node[{}]",
|
||||||
|
activeData.getCid(),
|
||||||
|
serverData.getCid());
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
logger.warn("canal is not run any in node");
|
||||||
|
return false;
|
||||||
|
} catch (ZkInterruptedException e) {
|
||||||
|
logger.warn("canal check is interrupt");
|
||||||
|
Thread.interrupted();// 清除interrupt标记
|
||||||
|
return check();
|
||||||
|
} catch (ZkException e) {
|
||||||
|
logger.warn("canal check is failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean releaseRunning() {
|
||||||
|
if (check()) {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
|
||||||
|
zkClient.delete(path);
|
||||||
|
mutex.set(false);
|
||||||
|
processActiveExit();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ====================== helper method ======================
|
||||||
|
|
||||||
|
private boolean isMine(String address) {
|
||||||
|
return address.equals(serverData.getAddress());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processStart() {
|
||||||
|
if (listener != null) {
|
||||||
|
try {
|
||||||
|
listener.processStart();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("processStart failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processStop() {
|
||||||
|
if (listener != null) {
|
||||||
|
try {
|
||||||
|
listener.processStop();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("processStop failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processActiveEnter() {
|
||||||
|
if (listener != null) {
|
||||||
|
try {
|
||||||
|
listener.processActiveEnter();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("processActiveEnter failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processActiveExit() {
|
||||||
|
if (listener != null) {
|
||||||
|
try {
|
||||||
|
listener.processActiveExit();
|
||||||
|
} catch (Exception e) {
|
||||||
|
logger.error("processActiveExit failed", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setListener(ServerRunningListener listener) {
|
||||||
|
this.listener = listener;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===================== setter / getter =======================
|
||||||
|
|
||||||
|
public void setDelayTime(int delayTime) {
|
||||||
|
this.delayTime = delayTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setServerData(ServerRunningData serverData) {
|
||||||
|
this.serverData = serverData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDestination(String destination) {
|
||||||
|
this.destination = destination;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setZkClient(ZkClientx zkClient) {
|
||||||
|
this.zkClient = zkClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
package com.alibaba.otter.canal.common.zookeeper.running;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@linkplain ServerRunningMonitor}管理容器，使用static进行数据全局共享
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-12-3 下午09:32:06
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class ServerRunningMonitors {
|
||||||
|
|
||||||
|
private static ServerRunningData serverData;
|
||||||
|
private static Map runningMonitors; // <String, ServerRunningMonitor>
|
||||||
|
|
||||||
|
public static ServerRunningData getServerData() {
|
||||||
|
return serverData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Map<String, ServerRunningMonitor> getRunningMonitors() {
|
||||||
|
return runningMonitors;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ServerRunningMonitor getRunningMonitor(String destination) {
|
||||||
|
return (ServerRunningMonitor) runningMonitors.get(destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void setServerData(ServerRunningData serverData) {
|
||||||
|
ServerRunningMonitors.serverData = serverData;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void setRunningMonitors(Map runningMonitors) {
|
||||||
|
ServerRunningMonitors.runningMonitors = runningMonitors;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package com.alibaba.otter.canal.common;
|
||||||
|
|
||||||
|
import org.junit.Assert;
|
||||||
|
|
||||||
|
public class AbstractZkTest {
|
||||||
|
|
||||||
|
protected String destination = "ljhtest1";
|
||||||
|
protected String cluster1 = "127.0.0.1:2188";
|
||||||
|
protected String cluster2 = "127.0.0.1:2188,127.0.0.1:2188";
|
||||||
|
|
||||||
|
public void sleep(long time) {
|
||||||
|
try {
|
||||||
|
Thread.sleep(time);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Assert.fail(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package com.alibaba.otter.canal.common;
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.math.RandomUtils;
|
||||||
|
import org.junit.After;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
|
||||||
|
|
||||||
|
public class ServerRunningTest extends AbstractZkTest {
|
||||||
|
|
||||||
|
private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||||
|
zkclientx.deleteRecursive(path);
|
||||||
|
|
||||||
|
zkclientx.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() {
|
||||||
|
String path = ZookeeperPathUtils.getDestinationPath(destination);
|
||||||
|
zkclientx.deleteRecursive(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testOneServer() {
|
||||||
|
final CountDownLatch countLatch = new CountDownLatch(2);
|
||||||
|
ServerRunningMonitor runningMonitor = buildServerRunning(countLatch, 1L, "127.0.0.1", 2088);
|
||||||
|
runningMonitor.start();
|
||||||
|
sleep(2000L);
|
||||||
|
runningMonitor.stop();
|
||||||
|
sleep(2000L);
|
||||||
|
|
||||||
|
if (countLatch.getCount() != 0) {
|
||||||
|
Assert.fail();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMultiServer() {
|
||||||
|
final CountDownLatch countLatch = new CountDownLatch(30);
|
||||||
|
final ServerRunningMonitor runningMonitor1 = buildServerRunning(countLatch, 1L, "127.0.0.1", 2088);
|
||||||
|
final ServerRunningMonitor runningMonitor2 = buildServerRunning(countLatch, 2L, "127.0.0.1", 2089);
|
||||||
|
final ServerRunningMonitor runningMonitor3 = buildServerRunning(countLatch, 3L, "127.0.0.1", 2090);
|
||||||
|
final ExecutorService executor = Executors.newFixedThreadPool(3);
|
||||||
|
executor.submit(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
if (!runningMonitor1.isStart()) {
|
||||||
|
runningMonitor1.start();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
if (runningMonitor1.check()) {
|
||||||
|
runningMonitor1.stop();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
executor.submit(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
if (!runningMonitor2.isStart()) {
|
||||||
|
runningMonitor2.start();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
if (runningMonitor2.check()) {
|
||||||
|
runningMonitor2.stop();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
executor.submit(new Runnable() {
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
for (int i = 0; i < 10; i++) {
|
||||||
|
if (!runningMonitor3.isStart()) {
|
||||||
|
runningMonitor3.start();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
if (runningMonitor3.check()) {
|
||||||
|
runningMonitor3.stop();
|
||||||
|
}
|
||||||
|
sleep(2000L + RandomUtils.nextInt(500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
sleep(30000L);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ServerRunningMonitor buildServerRunning(final CountDownLatch countLatch, final Long cid, final String ip,
|
||||||
|
final int port) {
|
||||||
|
ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
|
||||||
|
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
|
||||||
|
runningMonitor.setDestination(destination);
|
||||||
|
runningMonitor.setListener(new ServerRunningListener() {
|
||||||
|
|
||||||
|
public void processActiveEnter() {
|
||||||
|
System.out.println(String.format("cid:%s ip:%s:%s has start", cid, ip, port));
|
||||||
|
countLatch.countDown();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processActiveExit() {
|
||||||
|
System.out.println(String.format("cid:%s ip:%s:%s has stop", cid, ip, port));
|
||||||
|
countLatch.countDown();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processStart() {
|
||||||
|
System.out.println(String.format("cid:%s ip:%s:%s processStart", cid, ip, port));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processStop() {
|
||||||
|
System.out.println(String.format("cid:%s ip:%s:%s processStop", cid, ip, port));
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
runningMonitor.setZkClient(zkclientx);
|
||||||
|
runningMonitor.setDelayTime(1);
|
||||||
|
return runningMonitor;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
<configuration scan="true" scanPeriod=" 5 seconds">
|
||||||
|
|
||||||
|
<jmxConfigurator />
|
||||||
|
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||||
|
<encoder>
|
||||||
|
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
|
||||||
|
</pattern>
|
||||||
|
</encoder>
|
||||||
|
</appender>
|
||||||
|
|
||||||
|
<root level="WARN">
|
||||||
|
<appender-ref ref="STDOUT"/>
|
||||||
|
</root>
|
||||||
|
</configuration>
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal</artifactId>
|
||||||
|
<version>1.0.19-SNAPSHOT</version>
|
||||||
|
<relativePath>../pom.xml</relativePath>
|
||||||
|
</parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal.parse.dbsync</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
<name>canal dbsync module for otter ${project.version}</name>
|
||||||
|
<dependencies>
|
||||||
|
<!-- log -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-core</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>ch.qos.logback</groupId>
|
||||||
|
<artifactId>logback-classic</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>jcl-over-slf4j</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-api</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<!-- test dependency -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>mysql</groupId>
|
||||||
|
<artifactId>mysql-connector-java</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</project>
|
||||||
@@ -0,0 +1,367 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.nio.charset.Charset;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An utility class implements MySQL/Java charsets conversion. you can see com.mysql.jdbc.CharsetMapping.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
*/
|
||||||
|
public final class CharsetConversion
|
||||||
|
{
|
||||||
|
static final Log logger = LogFactory.getLog(CharsetConversion.class);
|
||||||
|
|
||||||
|
static final class Entry
|
||||||
|
{
|
||||||
|
protected final int charsetId;
|
||||||
|
protected final String mysqlCharset;
|
||||||
|
protected final String mysqlCollation;
|
||||||
|
protected final String javaCharset;
|
||||||
|
|
||||||
|
Entry(final int id, String mysqlCharset, // NL
|
||||||
|
String mysqlCollation, String javaCharset)
|
||||||
|
{
|
||||||
|
this.charsetId = id;
|
||||||
|
this.mysqlCharset = mysqlCharset;
|
||||||
|
this.mysqlCollation = mysqlCollation;
|
||||||
|
this.javaCharset = javaCharset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Character set data used in lookups. The array will be sparse.
|
||||||
|
static final Entry[] entries = new Entry[0xff];
|
||||||
|
|
||||||
|
static Entry getEntry(final int id)
|
||||||
|
{
|
||||||
|
if (id >= 0 && id < entries.length)
|
||||||
|
{
|
||||||
|
return entries[id];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("Invalid charset id: " + id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loads character set information.
|
||||||
|
static void putEntry(final int charsetId, String mysqlCharset,
|
||||||
|
String mysqlCollation, String javaCharset)
|
||||||
|
{
|
||||||
|
entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
|
||||||
|
mysqlCollation, javaCharset);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loads character set information.
|
||||||
|
@Deprecated
|
||||||
|
static void putEntry(final int charsetId, String mysqlCharset,
|
||||||
|
String mysqlCollation)
|
||||||
|
{
|
||||||
|
entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
|
||||||
|
mysqlCollation, /* Unknown java charset */null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load character set data statically.
|
||||||
|
static
|
||||||
|
{
|
||||||
|
putEntry(1, "big5", "big5_chinese_ci", "Big5");
|
||||||
|
putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
|
||||||
|
putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
|
||||||
|
putEntry(4, "cp850", "cp850_general_ci", "Cp850");
|
||||||
|
putEntry(5, "latin1", "latin1_german1_ci", "ISO8859_1");
|
||||||
|
putEntry(6, "hp8", "hp8_english_ci", "ISO8859_1");
|
||||||
|
putEntry(7, "koi8r", "koi8r_general_ci", "KOI8_R");
|
||||||
|
putEntry(8, "latin1", "latin1_swedish_ci", "ISO8859_1");
|
||||||
|
putEntry(9, "latin2", "latin2_general_ci", "ISO8859_2");
|
||||||
|
putEntry(10, "swe7", "swe7_swedish_ci", "ISO8859_1");
|
||||||
|
putEntry(11, "ascii", "ascii_general_ci", "US-ASCII");
|
||||||
|
putEntry(12, "ujis", "ujis_japanese_ci", "EUC_JP");
|
||||||
|
putEntry(13, "sjis", "sjis_japanese_ci", "SJIS");
|
||||||
|
putEntry(14, "cp1251", "cp1251_bulgarian_ci", "Cp1251");
|
||||||
|
putEntry(15, "latin1", "latin1_danish_ci", "ISO8859_1");
|
||||||
|
putEntry(16, "hebrew", "hebrew_general_ci", "ISO8859_8");
|
||||||
|
putEntry(17, "filename", "filename", "ISO8859_1");
|
||||||
|
putEntry(18, "tis620", "tis620_thai_ci", "TIS620");
|
||||||
|
putEntry(19, "euckr", "euckr_korean_ci", "EUC_KR");
|
||||||
|
putEntry(20, "latin7", "latin7_estonian_cs", "ISO8859_7");
|
||||||
|
putEntry(21, "latin2", "latin2_hungarian_ci", "ISO8859_2");
|
||||||
|
putEntry(22, "koi8u", "koi8u_general_ci", "KOI8_U");
|
||||||
|
putEntry(23, "cp1251", "cp1251_ukrainian_ci", "Cp1251");
|
||||||
|
putEntry(24, "gb2312", "gb2312_chinese_ci", "EUC_CN");
|
||||||
|
putEntry(25, "greek", "greek_general_ci", "ISO8859_7");
|
||||||
|
putEntry(26, "cp1250", "cp1250_general_ci", "Cp1250");
|
||||||
|
putEntry(27, "latin2", "latin2_croatian_ci", "ISO8859_2");
|
||||||
|
putEntry(28, "gbk", "gbk_chinese_ci", "GBK");
|
||||||
|
putEntry(29, "cp1257", "cp1257_lithuanian_ci", "Cp1257");
|
||||||
|
putEntry(30, "latin5", "latin5_turkish_ci", "ISO8859_5");
|
||||||
|
putEntry(31, "latin1", "latin1_german2_ci", "ISO8859_1");
|
||||||
|
putEntry(32, "armscii8", "armscii8_general_ci", "ISO8859_1");
|
||||||
|
putEntry(33, "utf8", "utf8_general_ci", "UTF-8");
|
||||||
|
putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250");
|
||||||
|
putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig");
|
||||||
|
putEntry(36, "cp866", "cp866_general_ci", "Cp866");
|
||||||
|
putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp895");
|
||||||
|
putEntry(38, "macce", "macce_general_ci", "MacCentralEurope");
|
||||||
|
putEntry(39, "macroman", "macroman_general_ci", "MacRoman");
|
||||||
|
putEntry(40, "cp852", "cp852_general_ci", "Cp852");
|
||||||
|
putEntry(41, "latin7", "latin7_general_ci", "ISO8859_7");
|
||||||
|
putEntry(42, "latin7", "latin7_general_cs", "ISO8859_7");
|
||||||
|
putEntry(43, "macce", "macce_bin", "MacCentralEurope");
|
||||||
|
putEntry(44, "cp1250", "cp1250_croatian_ci", "Cp1250");
|
||||||
|
putEntry(45, "utf8mb4", "utf8mb4_general_ci", "MacCentralEurope");
|
||||||
|
putEntry(46, "utf8mb4", "utf8mb4_bin", "MacCentralEurope");
|
||||||
|
putEntry(47, "latin1", "latin1_bin", "ISO8859_1");
|
||||||
|
putEntry(48, "latin1", "latin1_general_ci", "ISO8859_1");
|
||||||
|
putEntry(49, "latin1", "latin1_general_cs", "ISO8859_1");
|
||||||
|
putEntry(50, "cp1251", "cp1251_bin", "Cp1251");
|
||||||
|
putEntry(51, "cp1251", "cp1251_general_ci", "Cp1251");
|
||||||
|
putEntry(52, "cp1251", "cp1251_general_cs", "Cp1251");
|
||||||
|
putEntry(53, "macroman", "macroman_bin", "MacRoman");
|
||||||
|
putEntry(54, "utf16", "utf16_general_ci", "UTF-16");
|
||||||
|
putEntry(55, "utf16", "utf16_bin", "UTF-16");
|
||||||
|
putEntry(57, "cp1256", "cp1256_general_ci", "Cp1256");
|
||||||
|
putEntry(58, "cp1257", "cp1257_bin", "Cp1257");
|
||||||
|
putEntry(59, "cp1257", "cp1257_general_ci", "Cp1257");
|
||||||
|
putEntry(60, "utf32", "utf32_general_ci", "UTF-32");
|
||||||
|
putEntry(61, "utf32", "utf32_bin", "UTF-32");
|
||||||
|
putEntry(63, "binary", "binary", "US-ASCII");
|
||||||
|
putEntry(64, "armscii8", "armscii8_bin", "ISO8859_2");
|
||||||
|
putEntry(65, "ascii", "ascii_bin", "US-ASCII");
|
||||||
|
putEntry(66, "cp1250", "cp1250_bin", "Cp1250");
|
||||||
|
putEntry(67, "cp1256", "cp1256_bin", "Cp1256");
|
||||||
|
putEntry(68, "cp866", "cp866_bin", "Cp866");
|
||||||
|
putEntry(69, "dec8", "dec8_bin", "US-ASCII");
|
||||||
|
putEntry(70, "greek", "greek_bin", "ISO8859_7");
|
||||||
|
putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
|
||||||
|
putEntry(72, "hp8", "hp8_bin" , "US-ASCII");
|
||||||
|
putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
|
||||||
|
putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
|
||||||
|
putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
|
||||||
|
putEntry(77, "latin2", "latin2_bin", "ISO8859_2");
|
||||||
|
putEntry(78, "latin5", "latin5_bin", "ISO8859_5");
|
||||||
|
putEntry(79, "latin7", "latin7_bin", "ISO8859_7");
|
||||||
|
putEntry(80, "cp850", "cp850_bin", "Cp850");
|
||||||
|
putEntry(81, "cp852", "cp852_bin", "Cp852");
|
||||||
|
putEntry(82, "swe7", "swe7_bin", "ISO8859_1");
|
||||||
|
putEntry(83, "utf8", "utf8_bin", "UTF-8");
|
||||||
|
putEntry(84, "big5", "big5_bin", "Big5");
|
||||||
|
putEntry(85, "euckr", "euckr_bin", "EUC_KR");
|
||||||
|
putEntry(86, "gb2312", "gb2312_bin", "EUC_CN");
|
||||||
|
putEntry(87, "gbk", "gbk_bin", "GBK");
|
||||||
|
putEntry(88, "sjis", "sjis_bin", "SJIS");
|
||||||
|
putEntry(89, "tis620", "tis620_bin", "TIS620");
|
||||||
|
putEntry(90, "ucs2", "ucs2_bin", "UnicodeBig");
|
||||||
|
putEntry(91, "ujis", "ujis_bin", "EUC_JP");
|
||||||
|
putEntry(92, "geostd8", "geostd8_general_ci", "US-ASCII");
|
||||||
|
putEntry(93, "geostd8", "geostd8_bin", "US-ASCII");
|
||||||
|
putEntry(94, "latin1", "latin1_spanish_ci", "ISO8859_1");
|
||||||
|
putEntry(95, "cp932", "cp932_japanese_ci", "Shift_JIS");
|
||||||
|
putEntry(96, "cp932", "cp932_bin", "Shift_JIS");
|
||||||
|
putEntry(97, "eucjpms", "eucjpms_japanese_ci", "EUC_JP");
|
||||||
|
putEntry(98, "eucjpms", "eucjpms_bin", "EUC_JP");
|
||||||
|
putEntry(99, "cp1250", "cp1250_polish_ci", "Cp1250");
|
||||||
|
|
||||||
|
putEntry(101, "utf16", "utf16_unicode_ci", "UTF-16");
|
||||||
|
putEntry(102, "utf16", "utf16_icelandic_ci", "UTF-16");
|
||||||
|
putEntry(103, "utf16", "utf16_latvian_ci", "UTF-16");
|
||||||
|
putEntry(104, "utf16", "utf16_romanian_ci", "UTF-16");
|
||||||
|
putEntry(105, "utf16", "utf16_slovenian_ci", "UTF-16");
|
||||||
|
putEntry(106, "utf16", "utf16_polish_ci", "UTF-16");
|
||||||
|
putEntry(107, "utf16", "utf16_estonian_ci", "UTF-16");
|
||||||
|
putEntry(108, "utf16", "utf16_spanish_ci", "UTF-16");
|
||||||
|
putEntry(109, "utf16", "utf16_swedish_ci", "UTF-16");
|
||||||
|
putEntry(110, "utf16", "utf16_turkish_ci", "UTF-16");
|
||||||
|
putEntry(111, "utf16", "utf16_czech_ci", "UTF-16");
|
||||||
|
putEntry(112, "utf16", "utf16_danish_ci", "UTF-16");
|
||||||
|
putEntry(113, "utf16", "utf16_lithuanian_ci", "UTF-16");
|
||||||
|
putEntry(114, "utf16", "utf16_slovak_ci", "UTF-16");
|
||||||
|
putEntry(115, "utf16", "utf16_spanish2_ci", "UTF-16");
|
||||||
|
putEntry(116, "utf16", "utf16_roman_ci", "UTF-16");
|
||||||
|
putEntry(117, "utf16", "utf16_persian_ci", "UTF-16");
|
||||||
|
putEntry(118, "utf16", "utf16_esperanto_ci", "UTF-16");
|
||||||
|
putEntry(119, "utf16", "utf16_hungarian_ci", "UTF-16");
|
||||||
|
putEntry(120, "utf16", "utf16_sinhala_ci", "UTF-16");
|
||||||
|
|
||||||
|
putEntry(128, "ucs2", "ucs2_unicode_ci", "UnicodeBig");
|
||||||
|
putEntry(129, "ucs2", "ucs2_icelandic_ci", "UnicodeBig");
|
||||||
|
putEntry(130, "ucs2", "ucs2_latvian_ci", "UnicodeBig");
|
||||||
|
putEntry(131, "ucs2", "ucs2_romanian_ci", "UnicodeBig");
|
||||||
|
putEntry(132, "ucs2", "ucs2_slovenian_ci", "UnicodeBig");
|
||||||
|
putEntry(133, "ucs2", "ucs2_polish_ci", "UnicodeBig");
|
||||||
|
putEntry(134, "ucs2", "ucs2_estonian_ci", "UnicodeBig");
|
||||||
|
putEntry(135, "ucs2", "ucs2_spanish_ci", "UnicodeBig");
|
||||||
|
putEntry(136, "ucs2", "ucs2_swedish_ci", "UnicodeBig");
|
||||||
|
putEntry(137, "ucs2", "ucs2_turkish_ci", "UnicodeBig");
|
||||||
|
putEntry(138, "ucs2", "ucs2_czech_ci", "UnicodeBig");
|
||||||
|
putEntry(139, "ucs2", "ucs2_danish_ci", "UnicodeBig");
|
||||||
|
putEntry(140, "ucs2", "ucs2_lithuanian_ci", "UnicodeBig");
|
||||||
|
putEntry(141, "ucs2", "ucs2_slovak_ci", "UnicodeBig");
|
||||||
|
putEntry(142, "ucs2", "ucs2_spanish2_ci", "UnicodeBig");
|
||||||
|
putEntry(143, "ucs2", "ucs2_roman_ci", "UnicodeBig");
|
||||||
|
putEntry(144, "ucs2", "ucs2_persian_ci", "UnicodeBig");
|
||||||
|
putEntry(145, "ucs2", "ucs2_esperanto_ci", "UnicodeBig");
|
||||||
|
putEntry(146, "ucs2", "ucs2_hungarian_ci", "UnicodeBig");
|
||||||
|
putEntry(147, "ucs2", "ucs2_sinhala_ci", "UnicodeBig");
|
||||||
|
|
||||||
|
putEntry(160, "utf32", "utf32_unicode_ci", "UTF-32");
|
||||||
|
putEntry(161, "utf32", "utf32_icelandic_ci", "UTF-32");
|
||||||
|
putEntry(162, "utf32", "utf32_latvian_ci", "UTF-32");
|
||||||
|
putEntry(163, "utf32", "utf32_romanian_ci", "UTF-32");
|
||||||
|
putEntry(164, "utf32", "utf32_slovenian_ci", "UTF-32");
|
||||||
|
putEntry(165, "utf32", "utf32_polish_ci", "UTF-32");
|
||||||
|
putEntry(166, "utf32", "utf32_estonian_ci", "UTF-32");
|
||||||
|
putEntry(167, "utf32", "utf32_spanish_ci", "UTF-32");
|
||||||
|
putEntry(168, "utf32", "utf32_swedish_ci", "UTF-32");
|
||||||
|
putEntry(169, "utf32", "utf32_turkish_ci", "UTF-32");
|
||||||
|
putEntry(170, "utf32", "utf32_czech_ci", "UTF-32");
|
||||||
|
putEntry(171, "utf32", "utf32_danish_ci", "UTF-32");
|
||||||
|
putEntry(172, "utf32", "utf32_lithuanian_ci", "UTF-32");
|
||||||
|
putEntry(173, "utf32", "utf32_slovak_ci", "UTF-32");
|
||||||
|
putEntry(174, "utf32", "utf32_spanish2_ci", "UTF-32");
|
||||||
|
putEntry(175, "utf32", "utf32_roman_ci", "UTF-32");
|
||||||
|
putEntry(176, "utf32", "utf32_persian_ci", "UTF-32");
|
||||||
|
putEntry(177, "utf32", "utf32_esperanto_ci", "UTF-32");
|
||||||
|
putEntry(178, "utf32", "utf32_hungarian_ci", "UTF-32");
|
||||||
|
putEntry(179, "utf32", "utf32_sinhala_ci", "UTF-32");
|
||||||
|
|
||||||
|
putEntry(192, "utf8", "utf8_unicode_ci", "UTF-8");
|
||||||
|
putEntry(193, "utf8", "utf8_icelandic_ci", "UTF-8");
|
||||||
|
putEntry(194, "utf8", "utf8_latvian_ci", "UTF-8");
|
||||||
|
putEntry(195, "utf8", "utf8_romanian_ci", "UTF-8");
|
||||||
|
putEntry(196, "utf8", "utf8_slovenian_ci", "UTF-8");
|
||||||
|
putEntry(197, "utf8", "utf8_polish_ci", "UTF-8");
|
||||||
|
putEntry(198, "utf8", "utf8_estonian_ci", "UTF-8");
|
||||||
|
putEntry(199, "utf8", "utf8_spanish_ci", "UTF-8");
|
||||||
|
putEntry(200, "utf8", "utf8_swedish_ci", "UTF-8");
|
||||||
|
putEntry(201, "utf8", "utf8_turkish_ci", "UTF-8");
|
||||||
|
putEntry(202, "utf8", "utf8_czech_ci", "UTF-8");
|
||||||
|
putEntry(203, "utf8", "utf8_danish_ci", "UTF-8");
|
||||||
|
putEntry(204, "utf8", "utf8_lithuanian_ci", "UTF-8");
|
||||||
|
putEntry(205, "utf8", "utf8_slovak_ci", "UTF-8");
|
||||||
|
putEntry(206, "utf8", "utf8_spanish2_ci", "UTF-8");
|
||||||
|
putEntry(207, "utf8", "utf8_roman_ci", "UTF-8");
|
||||||
|
putEntry(208, "utf8", "utf8_persian_ci", "UTF-8");
|
||||||
|
putEntry(209, "utf8", "utf8_esperanto_ci", "UTF-8");
|
||||||
|
putEntry(210, "utf8", "utf8_hungarian_ci", "UTF-8");
|
||||||
|
putEntry(211, "utf8", "utf8_sinhala_ci", "UTF-8");
|
||||||
|
|
||||||
|
putEntry(224, "utf8mb4", "utf8mb4_unicode_ci", "UTF-8");
|
||||||
|
putEntry(225, "utf8mb4", "utf8mb4_icelandic_ci", "UTF-8");
|
||||||
|
putEntry(226, "utf8mb4", "utf8mb4_latvian_ci", "UTF-8");
|
||||||
|
putEntry(227, "utf8mb4", "utf8mb4_romanian_ci", "UTF-8");
|
||||||
|
putEntry(228, "utf8mb4", "utf8mb4_slovenian_ci", "UTF-8");
|
||||||
|
putEntry(229, "utf8mb4", "utf8mb4_polish_ci", "UTF-8");
|
||||||
|
putEntry(230, "utf8mb4", "utf8mb4_estonian_ci", "UTF-8");
|
||||||
|
putEntry(231, "utf8mb4", "utf8mb4_spanish_ci", "UTF-8");
|
||||||
|
putEntry(232, "utf8mb4", "utf8mb4_swedish_ci", "UTF-8");
|
||||||
|
putEntry(233, "utf8mb4", "utf8mb4_turkish_ci", "UTF-8");
|
||||||
|
putEntry(234, "utf8mb4", "utf8mb4_czech_ci", "UTF-8");
|
||||||
|
putEntry(235, "utf8mb4", "utf8mb4_danish_ci", "UTF-8");
|
||||||
|
putEntry(236, "utf8mb4", "utf8mb4_lithuanian_ci", "UTF-8");
|
||||||
|
putEntry(237, "utf8mb4", "utf8mb4_slovak_ci", "UTF-8");
|
||||||
|
putEntry(238, "utf8mb4", "utf8mb4_spanish2_ci", "UTF-8");
|
||||||
|
putEntry(239, "utf8mb4", "utf8mb4_roman_ci", "UTF-8");
|
||||||
|
putEntry(240, "utf8mb4", "utf8mb4_persian_ci", "UTF-8");
|
||||||
|
putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8");
|
||||||
|
putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8");
|
||||||
|
putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8");
|
||||||
|
|
||||||
|
putEntry(254, "utf8", "utf8_general_cs", "UTF-8");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return defined charset name for mysql.
|
||||||
|
*/
|
||||||
|
public static String getCharset(final int id)
|
||||||
|
{
|
||||||
|
Entry entry = getEntry(id);
|
||||||
|
|
||||||
|
if (entry != null)
|
||||||
|
{
|
||||||
|
return entry.mysqlCharset;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.warn("Unexpect mysql charset: " + id);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return defined collaction name for mysql.
|
||||||
|
*/
|
||||||
|
public static String getCollation(final int id)
|
||||||
|
{
|
||||||
|
Entry entry = getEntry(id);
|
||||||
|
|
||||||
|
if (entry != null)
|
||||||
|
{
|
||||||
|
return entry.mysqlCollation;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.warn("Unexpect mysql charset: " + id);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return converted charset name for java.
|
||||||
|
*/
|
||||||
|
public static String getJavaCharset(final int id)
|
||||||
|
{
|
||||||
|
Entry entry = getEntry(id);
|
||||||
|
|
||||||
|
if (entry != null)
|
||||||
|
{
|
||||||
|
if (entry.javaCharset != null)
|
||||||
|
{
|
||||||
|
return entry.javaCharset;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.warn("Unknown java charset for: id = " + id
|
||||||
|
+ ", name = " + entry.mysqlCharset + ", coll = "
|
||||||
|
+ entry.mysqlCollation);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.warn("Unexpect mysql charset: " + id);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < entries.length; i++)
|
||||||
|
{
|
||||||
|
Entry entry = entries[i];
|
||||||
|
|
||||||
|
System.out.print(i);
|
||||||
|
System.out.print(',');
|
||||||
|
System.out.print(' ');
|
||||||
|
if (entry != null)
|
||||||
|
{
|
||||||
|
System.out.print(entry.mysqlCharset);
|
||||||
|
System.out.print(',');
|
||||||
|
System.out.print(' ');
|
||||||
|
System.out.print(entry.javaCharset);
|
||||||
|
if (entry.javaCharset != null)
|
||||||
|
{
|
||||||
|
System.out.print(',');
|
||||||
|
System.out.print(' ');
|
||||||
|
System.out.print(Charset.forName(entry.javaCharset).name());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
System.out.print("null");
|
||||||
|
}
|
||||||
|
System.out.println();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,487 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.InterruptedIOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.net.SocketTimeoutException;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO: Document It!!
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* DirectLogFetcher fetcher = new DirectLogFetcher();
|
||||||
|
* fetcher.open(conn, file, 0, 13);
|
||||||
|
*
|
||||||
|
* while (fetcher.fetch())
|
||||||
|
* {
|
||||||
|
* LogEvent event;
|
||||||
|
* do
|
||||||
|
* {
|
||||||
|
* event = decoder.decode(fetcher, context);
|
||||||
|
*
|
||||||
|
* // process log event.
|
||||||
|
* }
|
||||||
|
* while (event != null);
|
||||||
|
* }
|
||||||
|
* // connection closed.
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class DirectLogFetcher extends LogFetcher
|
||||||
|
{
|
||||||
|
protected static final Log logger = LogFactory.getLog(DirectLogFetcher.class);
|
||||||
|
|
||||||
|
/** Command to dump binlog */
|
||||||
|
public static final byte COM_BINLOG_DUMP = 18;
|
||||||
|
|
||||||
|
/** Packet header sizes */
|
||||||
|
public static final int NET_HEADER_SIZE = 4;
|
||||||
|
public static final int SQLSTATE_LENGTH = 5;
|
||||||
|
|
||||||
|
/** Packet offsets */
|
||||||
|
public static final int PACKET_LEN_OFFSET = 0;
|
||||||
|
public static final int PACKET_SEQ_OFFSET = 3;
|
||||||
|
|
||||||
|
/** Maximum packet length */
|
||||||
|
public static final int MAX_PACKET_LENGTH = (256 * 256 * 256 - 1);
|
||||||
|
|
||||||
|
/** BINLOG_DUMP options */
|
||||||
|
public static final int BINLOG_DUMP_NON_BLOCK = 1;
|
||||||
|
public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
|
||||||
|
|
||||||
|
private Connection conn;
|
||||||
|
private OutputStream mysqlOutput;
|
||||||
|
private InputStream mysqlInput;
|
||||||
|
|
||||||
|
public DirectLogFetcher()
|
||||||
|
{
|
||||||
|
super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DirectLogFetcher(final int initialCapacity)
|
||||||
|
{
|
||||||
|
super(initialCapacity, DEFAULT_GROWTH_FACTOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DirectLogFetcher(final int initialCapacity, final float growthFactor)
|
||||||
|
{
|
||||||
|
super(initialCapacity, growthFactor);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final Object unwrapConnection(Object conn, Class<?> connClazz)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
while (!connClazz.isInstance(conn))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
|
||||||
|
if (connProxy.isInstance(conn))
|
||||||
|
{
|
||||||
|
conn = invokeMethod(conn, connProxy, "getTargetConnection");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (ClassNotFoundException e)
|
||||||
|
{
|
||||||
|
// org.springframework.jdbc.datasource.ConnectionProxy not found.
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
|
||||||
|
if (connProxy.isInstance(conn))
|
||||||
|
{
|
||||||
|
conn = getDeclaredField(conn, connProxy, "_conn");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (ClassNotFoundException e)
|
||||||
|
{
|
||||||
|
// org.apache.commons.dbcp.DelegatingConnection not found.
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (conn instanceof java.sql.Wrapper)
|
||||||
|
{
|
||||||
|
Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
|
||||||
|
conn = ((java.sql.Wrapper) conn).unwrap(connIface);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (ClassNotFoundException e)
|
||||||
|
{
|
||||||
|
// com.mysql.jdbc.Connection not found.
|
||||||
|
}
|
||||||
|
catch (SQLException e)
|
||||||
|
{
|
||||||
|
logger.warn("Unwrap " + conn.getClass().getName() + " to "
|
||||||
|
+ connClazz.getName() + " failed: " + e.getMessage(), e);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return conn;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final Object invokeMethod(Object obj, Class<?> objClazz,
|
||||||
|
String name)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Method method = objClazz.getMethod(name, (Class<?>[]) null);
|
||||||
|
return method.invoke(obj, (Object[]) null);
|
||||||
|
}
|
||||||
|
catch (NoSuchMethodException e)
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("No such method: \'" + name
|
||||||
|
+ "\' @ " + objClazz.getName(), e);
|
||||||
|
}
|
||||||
|
catch (IllegalAccessException e)
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("Cannot invoke method: \'"
|
||||||
|
+ name + "\' @ " + objClazz.getName(), e);
|
||||||
|
}
|
||||||
|
catch (InvocationTargetException e)
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("Invoke method failed: \'"
|
||||||
|
+ name + "\' @ " + objClazz.getName(),
|
||||||
|
e.getTargetException());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final Object getDeclaredField(Object obj, Class<?> objClazz,
|
||||||
|
String name)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Field field = objClazz.getDeclaredField(name);
|
||||||
|
field.setAccessible(true);
|
||||||
|
return field.get(obj);
|
||||||
|
}
|
||||||
|
catch (NoSuchFieldException e)
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("No such field: \'" + name
|
||||||
|
+ "\' @ " + objClazz.getName(), e);
|
||||||
|
}
|
||||||
|
catch (IllegalAccessException e)
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("Cannot get field: \'" + name
|
||||||
|
+ "\' @ " + objClazz.getName(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect MySQL master to fetch binlog.
|
||||||
|
*/
|
||||||
|
public void open(Connection conn, String fileName, final int serverId)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect MySQL master to fetch binlog.
|
||||||
|
*/
|
||||||
|
public void open(Connection conn, String fileName, final int serverId,
|
||||||
|
boolean nonBlocking) throws IOException
|
||||||
|
{
|
||||||
|
open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect MySQL master to fetch binlog.
|
||||||
|
*/
|
||||||
|
public void open(Connection conn, String fileName, final long filePosition,
|
||||||
|
final int serverId) throws IOException
|
||||||
|
{
|
||||||
|
open(conn, fileName, filePosition, serverId, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect MySQL master to fetch binlog.
|
||||||
|
*/
|
||||||
|
public void open(Connection conn, String fileName, long filePosition,
|
||||||
|
final int serverId, boolean nonBlocking) throws IOException
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
this.conn = conn;
|
||||||
|
Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
|
||||||
|
Object unwrapConn = unwrapConnection(conn, connClazz);
|
||||||
|
if (unwrapConn == null)
|
||||||
|
{
|
||||||
|
throw new IOException("Unable to unwrap "
|
||||||
|
+ conn.getClass().getName()
|
||||||
|
+ " to com.mysql.jdbc.ConnectionImpl");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get underlying IO streams for network communications.
|
||||||
|
Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
|
||||||
|
if (connIo == null)
|
||||||
|
{
|
||||||
|
throw new IOException("Get null field:"
|
||||||
|
+ conn.getClass().getName() + "#io");
|
||||||
|
}
|
||||||
|
mysqlOutput = (OutputStream) getDeclaredField(connIo,
|
||||||
|
connIo.getClass(), "mysqlOutput");
|
||||||
|
mysqlInput = (InputStream) getDeclaredField(connIo,
|
||||||
|
connIo.getClass(), "mysqlInput");
|
||||||
|
|
||||||
|
if (filePosition == 0)
|
||||||
|
filePosition = BIN_LOG_HEADER_SIZE;
|
||||||
|
sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
|
||||||
|
position = 0;
|
||||||
|
}
|
||||||
|
catch (IOException e)
|
||||||
|
{
|
||||||
|
close(); /* Do cleanup */
|
||||||
|
logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
|
||||||
|
+ ", position = " + filePosition);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
catch (ClassNotFoundException e)
|
||||||
|
{
|
||||||
|
close(); /* Do cleanup */
|
||||||
|
throw new IOException(
|
||||||
|
"Unable to load com.mysql.jdbc.ConnectionImpl", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put a byte in the buffer.
|
||||||
|
*
|
||||||
|
* @param b the byte to put in the buffer
|
||||||
|
*/
|
||||||
|
protected final void putByte(byte b)
|
||||||
|
{
|
||||||
|
ensureCapacity(position + 1);
|
||||||
|
|
||||||
|
buffer[position++] = b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put 16-bit integer in the buffer.
|
||||||
|
*
|
||||||
|
* @param i16 the integer to put in the buffer
|
||||||
|
*/
|
||||||
|
protected final void putInt16(int i16)
|
||||||
|
{
|
||||||
|
ensureCapacity(position + 2);
|
||||||
|
|
||||||
|
byte[] buf = buffer;
|
||||||
|
buf[position++] = (byte) (i16 & 0xff);
|
||||||
|
buf[position++] = (byte) (i16 >>> 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put 32-bit integer in the buffer.
|
||||||
|
*
|
||||||
|
* @param i32 the integer to put in the buffer
|
||||||
|
*/
|
||||||
|
protected final void putInt32(long i32)
|
||||||
|
{
|
||||||
|
ensureCapacity(position + 4);
|
||||||
|
|
||||||
|
byte[] buf = buffer;
|
||||||
|
buf[position++] = (byte) (i32 & 0xff);
|
||||||
|
buf[position++] = (byte) (i32 >>> 8);
|
||||||
|
buf[position++] = (byte) (i32 >>> 16);
|
||||||
|
buf[position++] = (byte) (i32 >>> 24);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put a string in the buffer.
|
||||||
|
*
|
||||||
|
* @param s the value to put in the buffer
|
||||||
|
*/
|
||||||
|
protected final void putString(String s)
|
||||||
|
{
|
||||||
|
ensureCapacity(position + (s.length() * 2) + 1);
|
||||||
|
|
||||||
|
System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
|
||||||
|
position += s.length();
|
||||||
|
buffer[position++] = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected final void sendBinlogDump(String fileName,
|
||||||
|
final long filePosition, final int serverId, boolean nonBlocking)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
position = NET_HEADER_SIZE;
|
||||||
|
|
||||||
|
putByte(COM_BINLOG_DUMP);
|
||||||
|
putInt32(filePosition);
|
||||||
|
int binlog_flags = nonBlocking ? BINLOG_DUMP_NON_BLOCK : 0;
|
||||||
|
binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
|
||||||
|
putInt16(binlog_flags); // binlog_flags
|
||||||
|
putInt32(serverId); // slave's server-id
|
||||||
|
putString(fileName);
|
||||||
|
|
||||||
|
final byte[] buf = buffer;
|
||||||
|
final int len = position - NET_HEADER_SIZE;
|
||||||
|
buf[0] = (byte) (len & 0xff);
|
||||||
|
buf[1] = (byte) (len >>> 8);
|
||||||
|
buf[2] = (byte) (len >>> 16);
|
||||||
|
|
||||||
|
mysqlOutput.write(buffer, 0, position);
|
||||||
|
mysqlOutput.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
|
||||||
|
*/
|
||||||
|
public boolean fetch() throws IOException
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
// Fetching packet header from input.
|
||||||
|
if (!fetch0(0, NET_HEADER_SIZE))
|
||||||
|
{
|
||||||
|
logger.warn("Reached end of input stream while fetching header");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetching the first packet(may a multi-packet).
|
||||||
|
int netlen = getUint24(PACKET_LEN_OFFSET);
|
||||||
|
int netnum = getUint8(PACKET_SEQ_OFFSET);
|
||||||
|
if (!fetch0(NET_HEADER_SIZE, netlen))
|
||||||
|
{
|
||||||
|
logger.warn("Reached end of input stream: packet #" + netnum
|
||||||
|
+ ", len = " + netlen);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detecting error code.
|
||||||
|
final int mark = getUint8(NET_HEADER_SIZE);
|
||||||
|
if (mark != 0)
|
||||||
|
{
|
||||||
|
if (mark == 255) // error from master
|
||||||
|
{
|
||||||
|
// Indicates an error, for example trying to fetch from wrong
|
||||||
|
// binlog position.
|
||||||
|
position = NET_HEADER_SIZE + 1;
|
||||||
|
final int errno = getInt16();
|
||||||
|
String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
|
||||||
|
String errmsg = getFixString(limit - position);
|
||||||
|
throw new IOException("Received error packet:"
|
||||||
|
+ " errno = " + errno + ", sqlstate = " + sqlstate
|
||||||
|
+ " errmsg = " + errmsg);
|
||||||
|
}
|
||||||
|
else if (mark == 254)
|
||||||
|
{
|
||||||
|
// Indicates end of stream. It's not clear when this would
|
||||||
|
// be sent.
|
||||||
|
logger.warn("Received EOF packet from server, apparent"
|
||||||
|
+ " master disconnected.");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Should not happen.
|
||||||
|
throw new IOException("Unexpected response " + mark
|
||||||
|
+ " while fetching binlog: packet #" + netnum
|
||||||
|
+ ", len = " + netlen);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first packet is a multi-packet, concatenate the packets.
|
||||||
|
while (netlen == MAX_PACKET_LENGTH)
|
||||||
|
{
|
||||||
|
if (!fetch0(0, NET_HEADER_SIZE))
|
||||||
|
{
|
||||||
|
logger.warn("Reached end of input stream while fetching header");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
netlen = getUint24(PACKET_LEN_OFFSET);
|
||||||
|
netnum = getUint8(PACKET_SEQ_OFFSET);
|
||||||
|
if (!fetch0(limit, netlen))
|
||||||
|
{
|
||||||
|
logger.warn("Reached end of input stream: packet #"
|
||||||
|
+ netnum + ", len = " + netlen);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preparing buffer variables to decoding.
|
||||||
|
origin = NET_HEADER_SIZE + 1;
|
||||||
|
position = origin;
|
||||||
|
limit -= origin;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
catch (SocketTimeoutException e)
|
||||||
|
{
|
||||||
|
close(); /* Do cleanup */
|
||||||
|
logger.error("Socket timeout expired, closing connection", e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
catch (InterruptedIOException e)
|
||||||
|
{
|
||||||
|
close(); /* Do cleanup */
|
||||||
|
logger.warn("I/O interrupted while reading from client socket", e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
catch (IOException e)
|
||||||
|
{
|
||||||
|
close(); /* Do cleanup */
|
||||||
|
logger.error("I/O error while reading from client socket", e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final boolean fetch0(final int off, final int len)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
ensureCapacity(off + len);
|
||||||
|
|
||||||
|
for (int count, n = 0; n < len; n += count)
|
||||||
|
{
|
||||||
|
if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
|
||||||
|
{
|
||||||
|
// Reached end of input stream
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (limit < off + len)
|
||||||
|
limit = off + len;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
|
||||||
|
*/
|
||||||
|
public void close() throws IOException
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (conn != null)
|
||||||
|
conn.close();
|
||||||
|
|
||||||
|
conn = null;
|
||||||
|
mysqlInput = null;
|
||||||
|
mysqlOutput = null;
|
||||||
|
}
|
||||||
|
catch (SQLException e)
|
||||||
|
{
|
||||||
|
logger.warn("Unable to close connection", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO: Document It!!
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* FileLogFetcher fetcher = new FileLogFetcher();
|
||||||
|
* fetcher.open(file, 0);
|
||||||
|
*
|
||||||
|
* while (fetcher.fetch()) {
|
||||||
|
* LogEvent event;
|
||||||
|
* do {
|
||||||
|
* event = decoder.decode(fetcher, context);
|
||||||
|
*
|
||||||
|
* // process log event.
|
||||||
|
* } while (event != null);
|
||||||
|
* }
|
||||||
|
* // file ending reached.
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class FileLogFetcher extends LogFetcher {
|
||||||
|
|
||||||
|
public static final byte[] BINLOG_MAGIC = { -2, 0x62, 0x69, 0x6e };
|
||||||
|
|
||||||
|
private FileInputStream fin;
|
||||||
|
|
||||||
|
public FileLogFetcher(){
|
||||||
|
super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
public FileLogFetcher(final int initialCapacity){
|
||||||
|
super(initialCapacity, DEFAULT_GROWTH_FACTOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
public FileLogFetcher(final int initialCapacity, final float growthFactor){
|
||||||
|
super(initialCapacity, growthFactor);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open binlog file in local disk to fetch.
|
||||||
|
*/
|
||||||
|
public void open(File file) throws FileNotFoundException, IOException {
|
||||||
|
open(file, 0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open binlog file in local disk to fetch.
|
||||||
|
*/
|
||||||
|
public void open(String filePath) throws FileNotFoundException, IOException {
|
||||||
|
open(new File(filePath), 0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open binlog file in local disk to fetch.
|
||||||
|
*/
|
||||||
|
public void open(String filePath, final long filePosition) throws FileNotFoundException, IOException {
|
||||||
|
open(new File(filePath), filePosition);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open binlog file in local disk to fetch.
|
||||||
|
*/
|
||||||
|
public void open(File file, final long filePosition) throws FileNotFoundException, IOException {
|
||||||
|
fin = new FileInputStream(file);
|
||||||
|
|
||||||
|
ensureCapacity(BIN_LOG_HEADER_SIZE);
|
||||||
|
if (BIN_LOG_HEADER_SIZE != fin.read(buffer, 0, BIN_LOG_HEADER_SIZE)) throw new IOException("No binlog file header");
|
||||||
|
|
||||||
|
if (buffer[0] != BINLOG_MAGIC[0] || buffer[1] != BINLOG_MAGIC[1] || buffer[2] != BINLOG_MAGIC[2]
|
||||||
|
|| buffer[3] != BINLOG_MAGIC[3]) {
|
||||||
|
throw new IOException("Error binlog file header: "
|
||||||
|
+ Arrays.toString(Arrays.copyOf(buffer, BIN_LOG_HEADER_SIZE)));
|
||||||
|
}
|
||||||
|
|
||||||
|
limit = 0;
|
||||||
|
origin = 0;
|
||||||
|
position = 0;
|
||||||
|
|
||||||
|
if (filePosition > BIN_LOG_HEADER_SIZE) {
|
||||||
|
final int maxFormatDescriptionEventLen = FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN
|
||||||
|
+ FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET
|
||||||
|
+ LogEvent.ENUM_END_EVENT + LogEvent.BINLOG_CHECKSUM_ALG_DESC_LEN
|
||||||
|
+ LogEvent.CHECKSUM_CRC32_SIGNATURE_LEN;
|
||||||
|
|
||||||
|
ensureCapacity(maxFormatDescriptionEventLen);
|
||||||
|
limit = fin.read(buffer, 0, maxFormatDescriptionEventLen);
|
||||||
|
limit = (int) getUint32(LogEvent.EVENT_LEN_OFFSET);
|
||||||
|
fin.getChannel().position(filePosition);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
|
||||||
|
*/
|
||||||
|
public boolean fetch() throws IOException {
|
||||||
|
if (limit == 0) {
|
||||||
|
final int len = fin.read(buffer, 0, buffer.length);
|
||||||
|
if (len >= 0) {
|
||||||
|
limit += len;
|
||||||
|
position = 0;
|
||||||
|
origin = 0;
|
||||||
|
|
||||||
|
/* More binlog to fetch */
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else if (origin == 0) {
|
||||||
|
if (limit > buffer.length / 2) {
|
||||||
|
ensureCapacity(buffer.length + limit);
|
||||||
|
}
|
||||||
|
final int len = fin.read(buffer, limit, buffer.length - limit);
|
||||||
|
if (len >= 0) {
|
||||||
|
limit += len;
|
||||||
|
|
||||||
|
/* More binlog to fetch */
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else if (limit > 0) {
|
||||||
|
System.arraycopy(buffer, origin, buffer, 0, limit);
|
||||||
|
final int len = fin.read(buffer, limit, buffer.length - limit);
|
||||||
|
if (len >= 0) {
|
||||||
|
limit += len;
|
||||||
|
position -= origin;
|
||||||
|
origin = 0;
|
||||||
|
|
||||||
|
/* More binlog to fetch */
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/* Should not happen. */
|
||||||
|
throw new IllegalArgumentException("Unexcepted limit: " + limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Reach binlog file end */
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
|
||||||
|
*/
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (fin != null) fin.close();
|
||||||
|
|
||||||
|
fin = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,77 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO: Document Me!!
|
||||||
|
*
|
||||||
|
* NOTE: Log context will NOT write multi-threaded.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class LogContext
|
||||||
|
{
|
||||||
|
private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
|
||||||
|
|
||||||
|
private FormatDescriptionLogEvent formatDescription;
|
||||||
|
|
||||||
|
private LogPosition logPosition;
|
||||||
|
|
||||||
|
public LogContext()
|
||||||
|
{
|
||||||
|
this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogContext(FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
this.formatDescription = descriptionEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final LogPosition getLogPosition()
|
||||||
|
{
|
||||||
|
return logPosition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void setLogPosition(LogPosition logPosition)
|
||||||
|
{
|
||||||
|
this.logPosition = logPosition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final FormatDescriptionLogEvent getFormatDescription()
|
||||||
|
{
|
||||||
|
return formatDescription;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void setFormatDescription(
|
||||||
|
FormatDescriptionLogEvent formatDescription)
|
||||||
|
{
|
||||||
|
this.formatDescription = formatDescription;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void putTable(TableMapLogEvent mapEvent)
|
||||||
|
{
|
||||||
|
mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final TableMapLogEvent getTable(final long tableId)
|
||||||
|
{
|
||||||
|
return mapOfTable.get(Long.valueOf(tableId));
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void clearAllTables()
|
||||||
|
{
|
||||||
|
mapOfTable.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reset()
|
||||||
|
{
|
||||||
|
formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
|
||||||
|
|
||||||
|
mapOfTable.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,494 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.BitSet;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.AppendBlockLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.BeginLoadQueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.CreateFileLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.DeleteFileLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.ExecuteLoadLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.ExecuteLoadQueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IncidentLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LoadLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.PreviousGtidsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.StartLogEventV3;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.StopLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implements a binary-log decoder.
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* LogDecoder decoder = new LogDecoder();
|
||||||
|
* decoder.handle(...);
|
||||||
|
*
|
||||||
|
* LogEvent event;
|
||||||
|
* do
|
||||||
|
* {
|
||||||
|
* event = decoder.decode(buffer, context);
|
||||||
|
*
|
||||||
|
* // process log event.
|
||||||
|
* }
|
||||||
|
* while (event != null);
|
||||||
|
* // no more events in buffer.
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class LogDecoder
|
||||||
|
{
|
||||||
|
protected static final Log logger = LogFactory.getLog(LogDecoder.class);
|
||||||
|
|
||||||
|
protected final BitSet handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
|
||||||
|
|
||||||
|
public LogDecoder()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogDecoder(final int fromIndex, final int toIndex)
|
||||||
|
{
|
||||||
|
handleSet.set(fromIndex, toIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void handle(final int fromIndex, final int toIndex)
|
||||||
|
{
|
||||||
|
handleSet.set(fromIndex, toIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void handle(final int flagIndex)
|
||||||
|
{
|
||||||
|
handleSet.set(flagIndex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decoding an event from binary-log buffer.
|
||||||
|
*
|
||||||
|
* @return <code>UknownLogEvent</code> if event type is unknown or skipped,
|
||||||
|
* <code>null</code> if buffer is not including a full event.
|
||||||
|
*/
|
||||||
|
public LogEvent decode(LogBuffer buffer, LogContext context)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
final int limit = buffer.limit();
|
||||||
|
|
||||||
|
if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
|
||||||
|
{
|
||||||
|
LogHeader header = new LogHeader(buffer,
|
||||||
|
context.getFormatDescription());
|
||||||
|
|
||||||
|
final int len = header.getEventLen();
|
||||||
|
if (limit >= len)
|
||||||
|
{
|
||||||
|
LogEvent event;
|
||||||
|
|
||||||
|
/* Checking binary-log's header */
|
||||||
|
if (handleSet.get(header.getType()))
|
||||||
|
{
|
||||||
|
buffer.limit(len);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
/* Decoding binary-log to event */
|
||||||
|
event = decode(buffer, header, context);
|
||||||
|
}
|
||||||
|
catch (IOException e)
|
||||||
|
{
|
||||||
|
if (logger.isWarnEnabled())
|
||||||
|
logger.warn("Decoding "
|
||||||
|
+ LogEvent.getTypeName(header.getType())
|
||||||
|
+ " failed from: "
|
||||||
|
+ context.getLogPosition(), e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
buffer.limit(limit); /* Restore limit */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
/* Ignore unsupported binary-log. */
|
||||||
|
event = new UnknownLogEvent(header);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* consume this binary-log. */
|
||||||
|
buffer.consume(len);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Rewind buffer's position to 0. */
|
||||||
|
buffer.rewind();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deserialize an event from buffer.
|
||||||
|
*
|
||||||
|
* @return <code>UknownLogEvent</code> if event type is unknown or skipped.
|
||||||
|
*/
|
||||||
|
public static LogEvent decode(LogBuffer buffer, LogHeader header,
|
||||||
|
LogContext context) throws IOException
|
||||||
|
{
|
||||||
|
FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
|
||||||
|
LogPosition logPosition = context.getLogPosition();
|
||||||
|
|
||||||
|
int checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
|
||||||
|
if (header.getType() != LogEvent.FORMAT_DESCRIPTION_EVENT) {
|
||||||
|
checksumAlg = descriptionEvent.header.getChecksumAlg();
|
||||||
|
}else {
|
||||||
|
// 如果是format事件自己，也需要处理checksum
|
||||||
|
checksumAlg = header.getChecksumAlg();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
|
||||||
|
// remove checksum bytes
|
||||||
|
buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (header.getType())
|
||||||
|
{
|
||||||
|
case LogEvent.QUERY_EVENT:
|
||||||
|
{
|
||||||
|
QueryLogEvent event = new QueryLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.XID_EVENT:
|
||||||
|
{
|
||||||
|
XidLogEvent event = new XidLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.TABLE_MAP_EVENT:
|
||||||
|
{
|
||||||
|
TableMapLogEvent mapEvent = new TableMapLogEvent(header,
|
||||||
|
buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
context.putTable(mapEvent);
|
||||||
|
return mapEvent;
|
||||||
|
}
|
||||||
|
case LogEvent.WRITE_ROWS_EVENT_V1:
|
||||||
|
{
|
||||||
|
RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
event.fillTable(context);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.UPDATE_ROWS_EVENT_V1:
|
||||||
|
{
|
||||||
|
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
event.fillTable(context);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.DELETE_ROWS_EVENT_V1:
|
||||||
|
{
|
||||||
|
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
event.fillTable(context);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.ROTATE_EVENT:
|
||||||
|
{
|
||||||
|
RotateLogEvent event = new RotateLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition = new LogPosition(event.getFilename(),
|
||||||
|
event.getPosition());
|
||||||
|
context.setLogPosition(logPosition);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.LOAD_EVENT:
|
||||||
|
case LogEvent.NEW_LOAD_EVENT:
|
||||||
|
{
|
||||||
|
LoadLogEvent event = new LoadLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
|
||||||
|
{
|
||||||
|
if (logger.isWarnEnabled())
|
||||||
|
logger.warn("Skipping unsupported SLAVE_EVENT from: "
|
||||||
|
+ context.getLogPosition());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.CREATE_FILE_EVENT:
|
||||||
|
{
|
||||||
|
CreateFileLogEvent event = new CreateFileLogEvent(header,
|
||||||
|
buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.APPEND_BLOCK_EVENT:
|
||||||
|
{
|
||||||
|
AppendBlockLogEvent event = new AppendBlockLogEvent(header,
|
||||||
|
buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.DELETE_FILE_EVENT:
|
||||||
|
{
|
||||||
|
DeleteFileLogEvent event = new DeleteFileLogEvent(header,
|
||||||
|
buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.EXEC_LOAD_EVENT:
|
||||||
|
{
|
||||||
|
ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
|
||||||
|
buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.START_EVENT_V3:
|
||||||
|
{
|
||||||
|
/* This is sent only by MySQL <=4.x */
|
||||||
|
StartLogEventV3 event = new StartLogEventV3(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.STOP_EVENT:
|
||||||
|
{
|
||||||
|
StopLogEvent event = new StopLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.INTVAR_EVENT:
|
||||||
|
{
|
||||||
|
IntvarLogEvent event = new IntvarLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.RAND_EVENT:
|
||||||
|
{
|
||||||
|
RandLogEvent event = new RandLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.USER_VAR_EVENT:
|
||||||
|
{
|
||||||
|
UserVarLogEvent event = new UserVarLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.FORMAT_DESCRIPTION_EVENT:
|
||||||
|
{
|
||||||
|
descriptionEvent = new FormatDescriptionLogEvent(header,
|
||||||
|
buffer, descriptionEvent);
|
||||||
|
context.setFormatDescription(descriptionEvent);
|
||||||
|
return descriptionEvent;
|
||||||
|
}
|
||||||
|
case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
|
||||||
|
{
|
||||||
|
if (logger.isWarnEnabled())
|
||||||
|
logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
|
||||||
|
+ context.getLogPosition());
|
||||||
|
// ev = new Write_rows_log_event_old(buf, event_len,
|
||||||
|
// description_event);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
|
||||||
|
{
|
||||||
|
if (logger.isWarnEnabled())
|
||||||
|
logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
|
||||||
|
+ context.getLogPosition());
|
||||||
|
// ev = new Update_rows_log_event_old(buf, event_len,
|
||||||
|
// description_event);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
|
||||||
|
{
|
||||||
|
if (logger.isWarnEnabled())
|
||||||
|
logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
|
||||||
|
+ context.getLogPosition());
|
||||||
|
// ev = new Delete_rows_log_event_old(buf, event_len,
|
||||||
|
// description_event);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.BEGIN_LOAD_QUERY_EVENT:
|
||||||
|
{
|
||||||
|
BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
|
||||||
|
header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
|
||||||
|
{
|
||||||
|
ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
|
||||||
|
header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.INCIDENT_EVENT:
|
||||||
|
{
|
||||||
|
IncidentLogEvent event = new IncidentLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.HEARTBEAT_LOG_EVENT:
|
||||||
|
{
|
||||||
|
HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
|
||||||
|
descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.IGNORABLE_LOG_EVENT:
|
||||||
|
{
|
||||||
|
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.ROWS_QUERY_LOG_EVENT:
|
||||||
|
{
|
||||||
|
RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.WRITE_ROWS_EVENT: {
|
||||||
|
RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
event.fillTable(context);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.UPDATE_ROWS_EVENT: {
|
||||||
|
RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
event.fillTable(context);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.DELETE_ROWS_EVENT: {
|
||||||
|
RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
event.fillTable(context);
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.GTID_LOG_EVENT:
|
||||||
|
case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
|
||||||
|
{
|
||||||
|
GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
|
||||||
|
{
|
||||||
|
PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.ANNOTATE_ROWS_EVENT:
|
||||||
|
{
|
||||||
|
AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.BINLOG_CHECKPOINT_EVENT:
|
||||||
|
{
|
||||||
|
BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.GTID_EVENT:
|
||||||
|
{
|
||||||
|
MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
case LogEvent.GTID_LIST_EVENT:
|
||||||
|
{
|
||||||
|
MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
/*
|
||||||
|
Create an object of Ignorable_log_event for unrecognized sub-class.
|
||||||
|
So that SLAVE SQL THREAD will only update the position and continue.
|
||||||
|
*/
|
||||||
|
if((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0){
|
||||||
|
IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
return event;
|
||||||
|
}else {
|
||||||
|
if (logger.isWarnEnabled())
|
||||||
|
logger.warn("Skipping unrecognized binlog event "
|
||||||
|
+ LogEvent.getTypeName(header.getType()) + " from: "
|
||||||
|
+ context.getLogPosition());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* updating position in context */
|
||||||
|
logPosition.position = header.getLogPos();
|
||||||
|
/* Unknown or unsupported log event */
|
||||||
|
return new UnknownLogEvent(header);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,437 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Binary log event definitions. This includes generic code common to all types
|
||||||
|
* of log events, as well as specific code for each type of log event.
|
||||||
|
*
|
||||||
|
* - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
|
||||||
|
* in little endian, i.e., the least significant byte first, unless otherwise
|
||||||
|
* specified.
|
||||||
|
*
|
||||||
|
* representation of unsigned integers, called Packed Integer. A Packed Integer
|
||||||
|
* has the capacity of storing up to 8-byte integers, while small integers still
|
||||||
|
* can use 1, 3, or 4 bytes. The value of the first byte determines how to read
|
||||||
|
* the number, according to the following table:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Format of Packed Integer</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>First byte</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>0-250</td>
|
||||||
|
* <td>The first byte is the number (in the range 0-250), and no more bytes are
|
||||||
|
* used.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>252</td>
|
||||||
|
* <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>253</td>
|
||||||
|
* <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>254</td>
|
||||||
|
* <td>Eight more bytes are used. The number is in the range
|
||||||
|
* 0xffffff-0xffffffffffffffff.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* - Strings are stored in various formats. The format of each string is
|
||||||
|
* documented separately.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.h
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public abstract class LogEvent
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* 3 is MySQL 4.x; 4 is MySQL 5.0.0.
|
||||||
|
* Compared to version 3, version 4 has:
|
||||||
|
* - a different Start_log_event, which includes info about the binary log
|
||||||
|
* (sizes of headers); this info is included for better compatibility if the
|
||||||
|
* master's MySQL version is different from the slave's.
|
||||||
|
* - all events have a unique ID (the triplet (server_id, timestamp at server
|
||||||
|
* start, other) to be sure an event is not executed more than once in a
|
||||||
|
* multimaster setup, example:
|
||||||
|
* M1
|
||||||
|
* / \
|
||||||
|
* v v
|
||||||
|
* M2 M3
|
||||||
|
* \ /
|
||||||
|
* v v
|
||||||
|
* S
|
||||||
|
* if a query is run on M1, it will arrive twice on S, so we need that S
|
||||||
|
* remembers the last unique ID it has processed, to compare and know if the
|
||||||
|
* event should be skipped or not. Example of ID: we already have the server id
|
||||||
|
* (4 bytes), plus:
|
||||||
|
* timestamp_when_the_master_started (4 bytes), a counter (a sequence number
|
||||||
|
* which increments every time we write an event to the binlog) (3 bytes).
|
||||||
|
* Q: how do we handle when the counter is overflowed and restarts from 0 ?
|
||||||
|
*
|
||||||
|
* - Query and Load (Create or Execute) events may have a more precise
|
||||||
|
* timestamp (with microseconds), number of matched/affected/warnings rows
|
||||||
|
* and fields of session variables: SQL_MODE,
|
||||||
|
* FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
|
||||||
|
* charsets, the PASSWORD() version (old/new/...).
|
||||||
|
*/
|
||||||
|
public static final int BINLOG_VERSION = 4;
|
||||||
|
|
||||||
|
/* Default 5.0 server version */
|
||||||
|
public static final String SERVER_VERSION = "5.0";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event header offsets; these point to places inside the fixed header.
|
||||||
|
*/
|
||||||
|
public static final int EVENT_TYPE_OFFSET = 4;
|
||||||
|
public static final int SERVER_ID_OFFSET = 5;
|
||||||
|
public static final int EVENT_LEN_OFFSET = 9;
|
||||||
|
public static final int LOG_POS_OFFSET = 13;
|
||||||
|
public static final int FLAGS_OFFSET = 17;
|
||||||
|
|
||||||
|
/* event-specific post-header sizes */
|
||||||
|
// where 3.23, 4.x and 5.0 agree
|
||||||
|
public static final int QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2);
|
||||||
|
// where 5.0 differs: 2 for len of N-bytes vars.
|
||||||
|
public static final int QUERY_HEADER_LEN = (QUERY_HEADER_MINIMAL_LEN + 2);
|
||||||
|
|
||||||
|
/* Enumeration type for the different types of log events. */
|
||||||
|
public static final int UNKNOWN_EVENT = 0;
|
||||||
|
public static final int START_EVENT_V3 = 1;
|
||||||
|
public static final int QUERY_EVENT = 2;
|
||||||
|
public static final int STOP_EVENT = 3;
|
||||||
|
public static final int ROTATE_EVENT = 4;
|
||||||
|
public static final int INTVAR_EVENT = 5;
|
||||||
|
public static final int LOAD_EVENT = 6;
|
||||||
|
public static final int SLAVE_EVENT = 7;
|
||||||
|
public static final int CREATE_FILE_EVENT = 8;
|
||||||
|
public static final int APPEND_BLOCK_EVENT = 9;
|
||||||
|
public static final int EXEC_LOAD_EVENT = 10;
|
||||||
|
public static final int DELETE_FILE_EVENT = 11;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer sql_ex,
|
||||||
|
* allowing multibyte TERMINATED BY etc; both types share the same class
|
||||||
|
* (Load_log_event)
|
||||||
|
*/
|
||||||
|
public static final int NEW_LOAD_EVENT = 12;
|
||||||
|
public static final int RAND_EVENT = 13;
|
||||||
|
public static final int USER_VAR_EVENT = 14;
|
||||||
|
public static final int FORMAT_DESCRIPTION_EVENT = 15;
|
||||||
|
public static final int XID_EVENT = 16;
|
||||||
|
public static final int BEGIN_LOAD_QUERY_EVENT = 17;
|
||||||
|
public static final int EXECUTE_LOAD_QUERY_EVENT = 18;
|
||||||
|
|
||||||
|
public static final int TABLE_MAP_EVENT = 19;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* These event numbers were used for 5.1.0 to 5.1.15 and are therefore
|
||||||
|
* obsolete.
|
||||||
|
*/
|
||||||
|
public static final int PRE_GA_WRITE_ROWS_EVENT = 20;
|
||||||
|
public static final int PRE_GA_UPDATE_ROWS_EVENT = 21;
|
||||||
|
public static final int PRE_GA_DELETE_ROWS_EVENT = 22;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* These event numbers are used from 5.1.16 and forward
|
||||||
|
*/
|
||||||
|
public static final int WRITE_ROWS_EVENT_V1 = 23;
|
||||||
|
public static final int UPDATE_ROWS_EVENT_V1 = 24;
|
||||||
|
public static final int DELETE_ROWS_EVENT_V1 = 25;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Something out of the ordinary happened on the master
|
||||||
|
*/
|
||||||
|
public static final int INCIDENT_EVENT = 26;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Heartbeat event to be send by master at its idle time to ensure master's online status to slave
|
||||||
|
*/
|
||||||
|
public static final int HEARTBEAT_LOG_EVENT = 27;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
|
||||||
|
* case there is code for handling it, but which can be ignored if it is not recognized.
|
||||||
|
*/
|
||||||
|
public static final int IGNORABLE_LOG_EVENT = 28;
|
||||||
|
public static final int ROWS_QUERY_LOG_EVENT = 29;
|
||||||
|
|
||||||
|
/** Version 2 of the Row events */
|
||||||
|
public static final int WRITE_ROWS_EVENT = 30;
|
||||||
|
public static final int UPDATE_ROWS_EVENT = 31;
|
||||||
|
public static final int DELETE_ROWS_EVENT = 32;
|
||||||
|
|
||||||
|
public static final int GTID_LOG_EVENT = 33;
|
||||||
|
public static final int ANONYMOUS_GTID_LOG_EVENT = 34;
|
||||||
|
|
||||||
|
public static final int PREVIOUS_GTIDS_LOG_EVENT = 35;
|
||||||
|
|
||||||
|
// mariaDb 5.5.34
|
||||||
|
/* New MySQL/Sun events are to be added right above this comment */
|
||||||
|
public static final int MYSQL_EVENTS_END = 36;
|
||||||
|
|
||||||
|
public static final int MARIA_EVENTS_BEGIN = 160;
|
||||||
|
/* New Maria event numbers start from here */
|
||||||
|
public static final int ANNOTATE_ROWS_EVENT = 160;
|
||||||
|
/*
|
||||||
|
Binlog checkpoint event. Used for XA crash recovery on the master, not used
|
||||||
|
in replication.
|
||||||
|
A binlog checkpoint event specifies a binlog file such that XA crash
|
||||||
|
recovery can start from that file - and it is guaranteed to find all XIDs
|
||||||
|
that are prepared in storage engines but not yet committed.
|
||||||
|
*/
|
||||||
|
public static final int BINLOG_CHECKPOINT_EVENT = 161;
|
||||||
|
/*
|
||||||
|
Gtid event. For global transaction ID, used to start a new event group,
|
||||||
|
instead of the old BEGIN query event, and also to mark stand-alone
|
||||||
|
events.
|
||||||
|
*/
|
||||||
|
public static final int GTID_EVENT = 162;
|
||||||
|
/*
|
||||||
|
Gtid list event. Logged at the start of every binlog, to record the
|
||||||
|
current replication state. This consists of the last GTID seen for
|
||||||
|
each replication domain.
|
||||||
|
*/
|
||||||
|
public static final int GTID_LIST_EVENT = 163;
|
||||||
|
|
||||||
|
/** end marker */
|
||||||
|
public static final int ENUM_END_EVENT = 164;
|
||||||
|
|
||||||
|
/**
|
||||||
|
1 byte length, 1 byte format
|
||||||
|
Length is total length in bytes, including 2 byte header
|
||||||
|
Length values 0 and 1 are currently invalid and reserved.
|
||||||
|
*/
|
||||||
|
public static final int EXTRA_ROW_INFO_LEN_OFFSET = 0;
|
||||||
|
public static final int EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
|
||||||
|
public static final int EXTRA_ROW_INFO_HDR_BYTES = 2;
|
||||||
|
public static final int EXTRA_ROW_INFO_MAX_PAYLOAD = (255 - EXTRA_ROW_INFO_HDR_BYTES);
|
||||||
|
|
||||||
|
// Events are without checksum though its generator
|
||||||
|
public static final int BINLOG_CHECKSUM_ALG_OFF = 0;
|
||||||
|
// is checksum-capable New Master (NM).
|
||||||
|
// CRC32 of zlib algorithm.
|
||||||
|
public static final int BINLOG_CHECKSUM_ALG_CRC32 = 1;
|
||||||
|
// the cut line: valid alg range is [1, 0x7f].
|
||||||
|
public static final int BINLOG_CHECKSUM_ALG_ENUM_END = 2;
|
||||||
|
// special value to tag undetermined yet checksum
|
||||||
|
public static final int BINLOG_CHECKSUM_ALG_UNDEF = 255;
|
||||||
|
// or events from checksum-unaware servers
|
||||||
|
|
||||||
|
public static final int CHECKSUM_CRC32_SIGNATURE_LEN = 4;
|
||||||
|
public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
|
||||||
|
/**
|
||||||
|
* defined statically while there is just one alg implemented
|
||||||
|
*/
|
||||||
|
public static final int BINLOG_CHECKSUM_LEN = CHECKSUM_CRC32_SIGNATURE_LEN;
|
||||||
|
|
||||||
|
/* MySQL or old MariaDB slave with no announced capability. */
|
||||||
|
public static final int MARIA_SLAVE_CAPABILITY_UNKNOWN = 0;
|
||||||
|
|
||||||
|
/* MariaDB >= 5.3, which understands ANNOTATE_ROWS_EVENT. */
|
||||||
|
public static final int MARIA_SLAVE_CAPABILITY_ANNOTATE = 1;
|
||||||
|
/*
|
||||||
|
* MariaDB >= 5.5. This version has the capability to tolerate events
|
||||||
|
* omitted from the binlog stream without breaking replication (MySQL slaves
|
||||||
|
* fail because they mis-compute the offsets into the master's binlog).
|
||||||
|
*/
|
||||||
|
public static final int MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES = 2;
|
||||||
|
/* MariaDB >= 10.0, which knows about binlog_checkpoint_log_event. */
|
||||||
|
public static final int MARIA_SLAVE_CAPABILITY_BINLOG_CHECKPOINT = 3;
|
||||||
|
/* MariaDB >= 10.0.1, which knows about global transaction id events. */
|
||||||
|
public static final int MARIA_SLAVE_CAPABILITY_GTID = 4;
|
||||||
|
|
||||||
|
/* Our capability. */
|
||||||
|
public static final int MARIA_SLAVE_CAPABILITY_MINE = MARIA_SLAVE_CAPABILITY_GTID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
For an event, 'e', carrying a type code, that a slave,
|
||||||
|
's', does not recognize, 's' will check 'e' for
|
||||||
|
LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
|
||||||
|
is ignored. Otherwise, 's' acknowledges that it has
|
||||||
|
found an unknown event in the relay log.
|
||||||
|
*/
|
||||||
|
public static final int LOG_EVENT_IGNORABLE_F = 0x80;
|
||||||
|
|
||||||
|
/** enum_field_types */
|
||||||
|
public static final int MYSQL_TYPE_DECIMAL = 0;
|
||||||
|
public static final int MYSQL_TYPE_TINY = 1;
|
||||||
|
public static final int MYSQL_TYPE_SHORT = 2;
|
||||||
|
public static final int MYSQL_TYPE_LONG = 3;
|
||||||
|
public static final int MYSQL_TYPE_FLOAT = 4;
|
||||||
|
public static final int MYSQL_TYPE_DOUBLE = 5;
|
||||||
|
public static final int MYSQL_TYPE_NULL = 6;
|
||||||
|
public static final int MYSQL_TYPE_TIMESTAMP = 7;
|
||||||
|
public static final int MYSQL_TYPE_LONGLONG = 8;
|
||||||
|
public static final int MYSQL_TYPE_INT24 = 9;
|
||||||
|
public static final int MYSQL_TYPE_DATE = 10;
|
||||||
|
public static final int MYSQL_TYPE_TIME = 11;
|
||||||
|
public static final int MYSQL_TYPE_DATETIME = 12;
|
||||||
|
public static final int MYSQL_TYPE_YEAR = 13;
|
||||||
|
public static final int MYSQL_TYPE_NEWDATE = 14;
|
||||||
|
public static final int MYSQL_TYPE_VARCHAR = 15;
|
||||||
|
public static final int MYSQL_TYPE_BIT = 16;
|
||||||
|
public static final int MYSQL_TYPE_TIMESTAMP2 = 17;
|
||||||
|
public static final int MYSQL_TYPE_DATETIME2 = 18;
|
||||||
|
public static final int MYSQL_TYPE_TIME2 = 19;
|
||||||
|
public static final int MYSQL_TYPE_NEWDECIMAL = 246;
|
||||||
|
public static final int MYSQL_TYPE_ENUM = 247;
|
||||||
|
public static final int MYSQL_TYPE_SET = 248;
|
||||||
|
public static final int MYSQL_TYPE_TINY_BLOB = 249;
|
||||||
|
public static final int MYSQL_TYPE_MEDIUM_BLOB = 250;
|
||||||
|
public static final int MYSQL_TYPE_LONG_BLOB = 251;
|
||||||
|
public static final int MYSQL_TYPE_BLOB = 252;
|
||||||
|
public static final int MYSQL_TYPE_VAR_STRING = 253;
|
||||||
|
public static final int MYSQL_TYPE_STRING = 254;
|
||||||
|
public static final int MYSQL_TYPE_GEOMETRY = 255;
|
||||||
|
|
||||||
|
public static String getTypeName(final int type)
|
||||||
|
{
|
||||||
|
switch (type)
|
||||||
|
{
|
||||||
|
case START_EVENT_V3:
|
||||||
|
return "Start_v3";
|
||||||
|
case STOP_EVENT:
|
||||||
|
return "Stop";
|
||||||
|
case QUERY_EVENT:
|
||||||
|
return "Query";
|
||||||
|
case ROTATE_EVENT:
|
||||||
|
return "Rotate";
|
||||||
|
case INTVAR_EVENT:
|
||||||
|
return "Intvar";
|
||||||
|
case LOAD_EVENT:
|
||||||
|
return "Load";
|
||||||
|
case NEW_LOAD_EVENT:
|
||||||
|
return "New_load";
|
||||||
|
case SLAVE_EVENT:
|
||||||
|
return "Slave";
|
||||||
|
case CREATE_FILE_EVENT:
|
||||||
|
return "Create_file";
|
||||||
|
case APPEND_BLOCK_EVENT:
|
||||||
|
return "Append_block";
|
||||||
|
case DELETE_FILE_EVENT:
|
||||||
|
return "Delete_file";
|
||||||
|
case EXEC_LOAD_EVENT:
|
||||||
|
return "Exec_load";
|
||||||
|
case RAND_EVENT:
|
||||||
|
return "RAND";
|
||||||
|
case XID_EVENT:
|
||||||
|
return "Xid";
|
||||||
|
case USER_VAR_EVENT:
|
||||||
|
return "User var";
|
||||||
|
case FORMAT_DESCRIPTION_EVENT:
|
||||||
|
return "Format_desc";
|
||||||
|
case TABLE_MAP_EVENT:
|
||||||
|
return "Table_map";
|
||||||
|
case PRE_GA_WRITE_ROWS_EVENT:
|
||||||
|
return "Write_rows_event_old";
|
||||||
|
case PRE_GA_UPDATE_ROWS_EVENT:
|
||||||
|
return "Update_rows_event_old";
|
||||||
|
case PRE_GA_DELETE_ROWS_EVENT:
|
||||||
|
return "Delete_rows_event_old";
|
||||||
|
case WRITE_ROWS_EVENT_V1:
|
||||||
|
return "Write_rows_v1";
|
||||||
|
case UPDATE_ROWS_EVENT_V1:
|
||||||
|
return "Update_rows_v1";
|
||||||
|
case DELETE_ROWS_EVENT_V1:
|
||||||
|
return "Delete_rows_v1";
|
||||||
|
case BEGIN_LOAD_QUERY_EVENT:
|
||||||
|
return "Begin_load_query";
|
||||||
|
case EXECUTE_LOAD_QUERY_EVENT:
|
||||||
|
return "Execute_load_query";
|
||||||
|
case INCIDENT_EVENT:
|
||||||
|
return "Incident";
|
||||||
|
case HEARTBEAT_LOG_EVENT:
|
||||||
|
return "Heartbeat";
|
||||||
|
case IGNORABLE_LOG_EVENT:
|
||||||
|
return "Ignorable";
|
||||||
|
case ROWS_QUERY_LOG_EVENT:
|
||||||
|
return "Rows_query";
|
||||||
|
case WRITE_ROWS_EVENT:
|
||||||
|
return "Write_rows";
|
||||||
|
case UPDATE_ROWS_EVENT:
|
||||||
|
return "Update_rows";
|
||||||
|
case DELETE_ROWS_EVENT:
|
||||||
|
return "Delete_rows";
|
||||||
|
case GTID_LOG_EVENT:
|
||||||
|
return "Gtid";
|
||||||
|
case ANONYMOUS_GTID_LOG_EVENT:
|
||||||
|
return "Anonymous_Gtid";
|
||||||
|
case PREVIOUS_GTIDS_LOG_EVENT:
|
||||||
|
return "Previous_gtids";
|
||||||
|
default:
|
||||||
|
return "Unknown"; /* impossible */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static final Log logger = LogFactory.getLog(LogEvent.class);
|
||||||
|
|
||||||
|
protected final LogHeader header;
|
||||||
|
|
||||||
|
protected LogEvent(LogHeader header)
|
||||||
|
{
|
||||||
|
this.header = header;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return event header.
|
||||||
|
*/
|
||||||
|
public final LogHeader getHeader()
|
||||||
|
{
|
||||||
|
return header;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The total size of this event, in bytes. In other words, this is the sum
|
||||||
|
* of the sizes of Common-Header, Post-Header, and Body.
|
||||||
|
*/
|
||||||
|
public final int getEventLen()
|
||||||
|
{
|
||||||
|
return header.getEventLen();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Server ID of the server that created the event.
|
||||||
|
*/
|
||||||
|
public final long getServerId()
|
||||||
|
{
|
||||||
|
return header.getServerId();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The position of the next event in the master binary log, in bytes from
|
||||||
|
* the beginning of the file. In a binlog that is not a relay log, this is
|
||||||
|
* just the position of the next event, in bytes from the beginning of the
|
||||||
|
* file. In a relay log, this is the position of the next event in the
|
||||||
|
* master's binlog.
|
||||||
|
*/
|
||||||
|
public final long getLogPos()
|
||||||
|
{
|
||||||
|
return header.getLogPos();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The time when the query started, in seconds since 1970.
|
||||||
|
*/
|
||||||
|
public final long getWhen()
|
||||||
|
{
|
||||||
|
return header.getWhen();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Declaration a binary-log fetcher. It extends from <code>LogBuffer</code>.
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* LogFetcher fetcher = new SomeLogFetcher();
|
||||||
|
* ...
|
||||||
|
*
|
||||||
|
* while (fetcher.fetch())
|
||||||
|
* {
|
||||||
|
* LogEvent event;
|
||||||
|
* do
|
||||||
|
* {
|
||||||
|
* event = decoder.decode(fetcher, context);
|
||||||
|
*
|
||||||
|
* // process log event.
|
||||||
|
* }
|
||||||
|
* while (event != null);
|
||||||
|
* }
|
||||||
|
* // no more binlog.
|
||||||
|
* fetcher.close();
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public abstract class LogFetcher extends LogBuffer implements Closeable
|
||||||
|
{
|
||||||
|
/** Default initial capacity. */
|
||||||
|
public static final int DEFAULT_INITIAL_CAPACITY = 8192;
|
||||||
|
|
||||||
|
/** Default growth factor. */
|
||||||
|
public static final float DEFAULT_GROWTH_FACTOR = 2.0f;
|
||||||
|
|
||||||
|
/** Binlog file header size */
|
||||||
|
public static final int BIN_LOG_HEADER_SIZE = 4;
|
||||||
|
|
||||||
|
protected final float factor;
|
||||||
|
|
||||||
|
public LogFetcher()
|
||||||
|
{
|
||||||
|
this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogFetcher(final int initialCapacity)
|
||||||
|
{
|
||||||
|
this(initialCapacity, DEFAULT_GROWTH_FACTOR);
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogFetcher(final int initialCapacity, final float growthFactor)
|
||||||
|
{
|
||||||
|
this.buffer = new byte[initialCapacity];
|
||||||
|
this.factor = growthFactor;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increases the capacity of this <tt>LogFetcher</tt> instance, if
|
||||||
|
* necessary, to ensure that it can hold at least the number of elements
|
||||||
|
* specified by the minimum capacity argument.
|
||||||
|
*
|
||||||
|
* @param minCapacity the desired minimum capacity
|
||||||
|
*/
|
||||||
|
protected final void ensureCapacity(final int minCapacity)
|
||||||
|
{
|
||||||
|
final int oldCapacity = buffer.length;
|
||||||
|
|
||||||
|
if (minCapacity > oldCapacity)
|
||||||
|
{
|
||||||
|
int newCapacity = (int) (oldCapacity * factor);
|
||||||
|
if (newCapacity < minCapacity)
|
||||||
|
newCapacity = minCapacity;
|
||||||
|
|
||||||
|
buffer = Arrays.copyOf(buffer, newCapacity);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetches the next frame of binary-log, and fill it in buffer.
|
||||||
|
*/
|
||||||
|
public abstract boolean fetch() throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see java.io.Closeable#close()
|
||||||
|
*/
|
||||||
|
public abstract void close() throws IOException;
|
||||||
|
}
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implements binlog position.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public class LogPosition implements Cloneable, Comparable<LogPosition>
|
||||||
|
{
|
||||||
|
/* binlog file's name */
|
||||||
|
protected String fileName;
|
||||||
|
|
||||||
|
/* position in file */
|
||||||
|
protected long position;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Binlog position init.
|
||||||
|
*
|
||||||
|
* @param fileName file name for binlog files: mysql-bin.000001
|
||||||
|
*/
|
||||||
|
public LogPosition(String fileName)
|
||||||
|
{
|
||||||
|
this.fileName = fileName;
|
||||||
|
this.position = 0L;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Binlog position init.
|
||||||
|
*
|
||||||
|
* @param fileName file name for binlog files: mysql-bin.000001
|
||||||
|
*/
|
||||||
|
public LogPosition(String fileName, final long position)
|
||||||
|
{
|
||||||
|
this.fileName = fileName;
|
||||||
|
this.position = position;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Binlog position copy init.
|
||||||
|
*/
|
||||||
|
public LogPosition(LogPosition source)
|
||||||
|
{
|
||||||
|
this.fileName = source.fileName;
|
||||||
|
this.position = source.position;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getFileName()
|
||||||
|
{
|
||||||
|
return fileName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getPosition()
|
||||||
|
{
|
||||||
|
return position;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Clone binlog position without CloneNotSupportedException */
|
||||||
|
public LogPosition clone()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return (LogPosition) super.clone();
|
||||||
|
}
|
||||||
|
catch (CloneNotSupportedException e)
|
||||||
|
{
|
||||||
|
// Never happend
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compares with the specified fileName and position.
|
||||||
|
*/
|
||||||
|
public final int compareTo(String fileName, final long position)
|
||||||
|
{
|
||||||
|
final int val = this.fileName.compareTo(fileName);
|
||||||
|
|
||||||
|
if (val == 0)
|
||||||
|
{
|
||||||
|
return (int) (this.position - position);
|
||||||
|
}
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see java.lang.Comparable#compareTo(java.lang.Object)
|
||||||
|
*/
|
||||||
|
public int compareTo(LogPosition o)
|
||||||
|
{
|
||||||
|
final int val = fileName.compareTo(o.fileName);
|
||||||
|
|
||||||
|
if (val == 0)
|
||||||
|
{
|
||||||
|
return (int) (position - o.position);
|
||||||
|
}
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see java.lang.Object#equals(java.lang.Object)
|
||||||
|
*/
|
||||||
|
public boolean equals(Object obj)
|
||||||
|
{
|
||||||
|
if (obj instanceof LogPosition)
|
||||||
|
{
|
||||||
|
LogPosition pos = ((LogPosition) obj);
|
||||||
|
return fileName.equals(pos.fileName)
|
||||||
|
&& (this.position == pos.position);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@inheritDoc}
|
||||||
|
*
|
||||||
|
* @see java.lang.Object#toString()
|
||||||
|
*/
|
||||||
|
public String toString()
|
||||||
|
{
|
||||||
|
return fileName + ':' + position;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append_block_log_event.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public class AppendBlockLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
private final LogBuffer blockBuf;
|
||||||
|
private final int blockLen;
|
||||||
|
|
||||||
|
private final long fileId;
|
||||||
|
|
||||||
|
/* AB = "Append Block" */
|
||||||
|
public static final int AB_FILE_ID_OFFSET = 0;
|
||||||
|
|
||||||
|
public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
|
||||||
|
final int totalHeaderLen = commonHeaderLen + postHeaderLen;
|
||||||
|
|
||||||
|
buffer.position(commonHeaderLen + AB_FILE_ID_OFFSET);
|
||||||
|
fileId = buffer.getUint32();
|
||||||
|
|
||||||
|
buffer.position(postHeaderLen);
|
||||||
|
blockLen = buffer.limit() - totalHeaderLen;
|
||||||
|
blockBuf = buffer.duplicate(blockLen);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getFileId()
|
||||||
|
{
|
||||||
|
return fileId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final LogBuffer getBuffer()
|
||||||
|
{
|
||||||
|
return blockBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final byte[] getData()
|
||||||
|
{
|
||||||
|
return blockBuf.getData();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event for the first block of file to be loaded, its only difference from
|
||||||
|
* Append_block event is that this event creates or truncates existing file
|
||||||
|
* before writing data.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
|
||||||
|
{
|
||||||
|
public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create_file_log_event.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class CreateFileLogEvent extends LoadLogEvent
|
||||||
|
{
|
||||||
|
protected LogBuffer blockBuf;
|
||||||
|
protected int blockLen;
|
||||||
|
protected long fileId;
|
||||||
|
|
||||||
|
protected boolean initedFromOld;
|
||||||
|
|
||||||
|
/* CF = "Create File" */
|
||||||
|
public static final int CF_FILE_ID_OFFSET = 0;
|
||||||
|
public static final int CF_DATA_OFFSET = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
|
||||||
|
|
||||||
|
public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
|
||||||
|
final int headerLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int loadHeaderLen = descriptionEvent.postHeaderLen[LOAD_EVENT - 1];
|
||||||
|
final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
|
||||||
|
|
||||||
|
copyLogEvent(buffer,
|
||||||
|
((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
|
||||||
|
: (headerLen + loadHeaderLen + createFileHeaderLen)),
|
||||||
|
descriptionEvent);
|
||||||
|
|
||||||
|
if (descriptionEvent.binlogVersion != 1)
|
||||||
|
{
|
||||||
|
fileId = buffer.getUint32(headerLen + loadHeaderLen
|
||||||
|
+ CF_FILE_ID_OFFSET);
|
||||||
|
/*
|
||||||
|
Note that it's ok to use get_data_size() below, because it is computed
|
||||||
|
with values we have already read from this event (because we called
|
||||||
|
copy_log_event()); we are not using slave's format info to decode
|
||||||
|
master's format, we are really using master's format info.
|
||||||
|
Anyway, both formats should be identical (except the common_header_len)
|
||||||
|
as these Load events are not changed between 4.0 and 5.0 (as logging of
|
||||||
|
LOAD DATA INFILE does not use Load_log_event in 5.0).
|
||||||
|
*/
|
||||||
|
blockLen = buffer.limit() - buffer.position();
|
||||||
|
blockBuf = buffer.duplicate(blockLen);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
initedFromOld = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getFileId()
|
||||||
|
{
|
||||||
|
return fileId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final LogBuffer getBuffer()
|
||||||
|
{
|
||||||
|
return blockBuf;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final byte[] getData()
|
||||||
|
{
|
||||||
|
return blockBuf.getData();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete_file_log_event.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class DeleteFileLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
private final long fileId;
|
||||||
|
|
||||||
|
/* DF = "Delete File" */
|
||||||
|
public static final int DF_FILE_ID_OFFSET = 0;
|
||||||
|
|
||||||
|
public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET);
|
||||||
|
fileId = buffer.getUint32(); // DF_FILE_ID_OFFSET
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getFileId()
|
||||||
|
{
|
||||||
|
return fileId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log row deletions. The event contain several delete rows for a table. Note
|
||||||
|
* that each event contains only rows for one table.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class DeleteRowsLogEvent extends RowsLogEvent
|
||||||
|
{
|
||||||
|
public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute_load_log_event.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class ExecuteLoadLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
private final long fileId;
|
||||||
|
|
||||||
|
/* EL = "Execute Load" */
|
||||||
|
public static final int EL_FILE_ID_OFFSET = 0;
|
||||||
|
|
||||||
|
public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET);
|
||||||
|
fileId = buffer.getUint32(); // EL_FILE_ID_OFFSET
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getFileId()
|
||||||
|
{
|
||||||
|
return fileId;
|
||||||
|
}
|
||||||
|
}
|
||||||
+101
@@ -0,0 +1,101 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Event responsible for LOAD DATA execution, it similar to Query_log_event but
|
||||||
|
* before executing the query it substitutes original filename in LOAD DATA
|
||||||
|
* query with name of temporary file.
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>4 bytes. The ID of the file to load.</li>
|
||||||
|
* <li>4 bytes. The start position within the statement for filename
|
||||||
|
* substitution.</li>
|
||||||
|
* <li>4 bytes. The end position within the statement for filename substitution.
|
||||||
|
* </li>
|
||||||
|
* <li>1 byte. How to handle duplicates: LOAD_DUP_ERROR = 0, LOAD_DUP_IGNORE =
|
||||||
|
* 1, LOAD_DUP_REPLACE = 2</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
|
||||||
|
{
|
||||||
|
/** file_id of temporary file */
|
||||||
|
private long fileId;
|
||||||
|
|
||||||
|
/** pointer to the part of the query that should be substituted */
|
||||||
|
private int fnPosStart;
|
||||||
|
|
||||||
|
/** pointer to the end of this part of query */
|
||||||
|
private int fnPosEnd;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Elements of this enum describe how LOAD DATA handles duplicates.
|
||||||
|
*/
|
||||||
|
public static final int LOAD_DUP_ERROR = 0;
|
||||||
|
public static final int LOAD_DUP_IGNORE = LOAD_DUP_ERROR + 1;
|
||||||
|
public static final int LOAD_DUP_REPLACE = LOAD_DUP_IGNORE + 1;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* We have to store type of duplicate handling explicitly, because for LOAD
|
||||||
|
* DATA it also depends on LOCAL option. And this part of query will be
|
||||||
|
* rewritten during replication so this information may be lost...
|
||||||
|
*/
|
||||||
|
private int dupHandling;
|
||||||
|
|
||||||
|
/* ELQ = "Execute Load Query" */
|
||||||
|
public static final int ELQ_FILE_ID_OFFSET = QUERY_HEADER_LEN;
|
||||||
|
public static final int ELQ_FN_POS_START_OFFSET = ELQ_FILE_ID_OFFSET + 4;
|
||||||
|
public static final int ELQ_FN_POS_END_OFFSET = ELQ_FILE_ID_OFFSET + 8;
|
||||||
|
public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
|
||||||
|
|
||||||
|
public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent) throws IOException
|
||||||
|
{
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
|
||||||
|
|
||||||
|
fileId = buffer.getUint32(); // ELQ_FILE_ID_OFFSET
|
||||||
|
fnPosStart = (int) buffer.getUint32(); // ELQ_FN_POS_START_OFFSET
|
||||||
|
fnPosEnd = (int) buffer.getUint32(); // ELQ_FN_POS_END_OFFSET
|
||||||
|
dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
|
||||||
|
|
||||||
|
final int len = query.length();
|
||||||
|
if (fnPosStart > len || fnPosEnd > len
|
||||||
|
|| dupHandling > LOAD_DUP_REPLACE)
|
||||||
|
{
|
||||||
|
throw new IOException(String.format(
|
||||||
|
"Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
|
||||||
|
+ "fn_pos_end=%d, dup_handling=%d", fnPosStart,
|
||||||
|
fnPosEnd, dupHandling));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getFilenamePosStart()
|
||||||
|
{
|
||||||
|
return fnPosStart;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getFilenamePosEnd()
|
||||||
|
{
|
||||||
|
return fnPosEnd;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getFilename()
|
||||||
|
{
|
||||||
|
if (query != null)
|
||||||
|
return query.substring(fnPosStart, fnPosEnd).trim();
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getFileId()
|
||||||
|
{
|
||||||
|
return fileId;
|
||||||
|
}
|
||||||
|
}
|
||||||
+305
@@ -0,0 +1,305 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
|
||||||
|
* ordinary events).
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class FormatDescriptionLogEvent extends StartLogEventV3 {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
|
||||||
|
* exist in binlogs, it does not have a format).
|
||||||
|
*/
|
||||||
|
public static final int LOG_EVENT_TYPES = (ENUM_END_EVENT - 1);
|
||||||
|
|
||||||
|
public static final int ST_COMMON_HEADER_LEN_OFFSET = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
|
||||||
|
|
||||||
|
public static final int OLD_HEADER_LEN = 13;
|
||||||
|
public static final int LOG_EVENT_HEADER_LEN = 19;
|
||||||
|
public static final int LOG_EVENT_MINIMAL_HEADER_LEN = 19;
|
||||||
|
|
||||||
|
/* event-specific post-header sizes */
|
||||||
|
public static final int STOP_HEADER_LEN = 0;
|
||||||
|
public static final int LOAD_HEADER_LEN = (4 + 4 + 4 + 1 + 1 + 4);
|
||||||
|
public static final int SLAVE_HEADER_LEN = 0;
|
||||||
|
public static final int START_V3_HEADER_LEN = (2 + ST_SERVER_VER_LEN + 4);
|
||||||
|
public static final int ROTATE_HEADER_LEN = 8; // this
|
||||||
|
// is
|
||||||
|
// FROZEN
|
||||||
|
// (the
|
||||||
|
// Rotate
|
||||||
|
// post-header
|
||||||
|
// is
|
||||||
|
// frozen)
|
||||||
|
public static final int INTVAR_HEADER_LEN = 0;
|
||||||
|
public static final int CREATE_FILE_HEADER_LEN = 4;
|
||||||
|
public static final int APPEND_BLOCK_HEADER_LEN = 4;
|
||||||
|
public static final int EXEC_LOAD_HEADER_LEN = 4;
|
||||||
|
public static final int DELETE_FILE_HEADER_LEN = 4;
|
||||||
|
public static final int NEW_LOAD_HEADER_LEN = LOAD_HEADER_LEN;
|
||||||
|
public static final int RAND_HEADER_LEN = 0;
|
||||||
|
public static final int USER_VAR_HEADER_LEN = 0;
|
||||||
|
public static final int FORMAT_DESCRIPTION_HEADER_LEN = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
|
||||||
|
public static final int XID_HEADER_LEN = 0;
|
||||||
|
public static final int BEGIN_LOAD_QUERY_HEADER_LEN = APPEND_BLOCK_HEADER_LEN;
|
||||||
|
public static final int ROWS_HEADER_LEN_V1 = 8;
|
||||||
|
public static final int TABLE_MAP_HEADER_LEN = 8;
|
||||||
|
public static final int EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
|
||||||
|
public static final int EXECUTE_LOAD_QUERY_HEADER_LEN = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
|
||||||
|
public static final int INCIDENT_HEADER_LEN = 2;
|
||||||
|
public static final int HEARTBEAT_HEADER_LEN = 0;
|
||||||
|
public static final int IGNORABLE_HEADER_LEN = 0;
|
||||||
|
public static final int ROWS_HEADER_LEN_V2 = 10;
|
||||||
|
public static final int ANNOTATE_ROWS_HEADER_LEN = 0;
|
||||||
|
public static final int BINLOG_CHECKPOINT_HEADER_LEN = 4;
|
||||||
|
public static final int GTID_HEADER_LEN = 19;
|
||||||
|
public static final int GTID_LIST_HEADER_LEN = 4;
|
||||||
|
|
||||||
|
public static final int POST_HEADER_LENGTH = 11;
|
||||||
|
|
||||||
|
public static final int BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
|
||||||
|
public static final int[] checksumVersionSplit = { 5, 6, 1 };
|
||||||
|
public static final long checksumVersionProduct = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
|
||||||
|
* 256 + checksumVersionSplit[2];
|
||||||
|
/**
|
||||||
|
* The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
|
||||||
|
* LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
|
||||||
|
* LOG_EVENT_MINIMAL_HEADER_LEN).
|
||||||
|
*/
|
||||||
|
protected final int commonHeaderLen;
|
||||||
|
protected int numberOfEventTypes;
|
||||||
|
|
||||||
|
/** The list of post-headers' lengthes */
|
||||||
|
protected final short[] postHeaderLen;
|
||||||
|
protected int[] serverVersionSplit = new int[3];
|
||||||
|
|
||||||
|
public FormatDescriptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
throws IOException{
|
||||||
|
/* Start_log_event_v3 */
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
|
||||||
|
buffer.position(LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET);
|
||||||
|
commonHeaderLen = buffer.getUint8();
|
||||||
|
if (commonHeaderLen < OLD_HEADER_LEN) /* sanity check */
|
||||||
|
{
|
||||||
|
throw new IOException("Format Description event header length is too short");
|
||||||
|
}
|
||||||
|
|
||||||
|
numberOfEventTypes = buffer.limit() - (LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET + 1);
|
||||||
|
|
||||||
|
// buffer.position(LOG_EVENT_MINIMAL_HEADER_LEN
|
||||||
|
// + ST_COMMON_HEADER_LEN_OFFSET + 1);
|
||||||
|
postHeaderLen = new short[numberOfEventTypes];
|
||||||
|
for (int i = 0; i < numberOfEventTypes; i++) {
|
||||||
|
postHeaderLen[i] = (short) buffer.getUint8();
|
||||||
|
}
|
||||||
|
|
||||||
|
calcServerVersionSplit();
|
||||||
|
long calc = getVersionProduct();
|
||||||
|
if (calc >= checksumVersionProduct) {
|
||||||
|
/* the last bytes are the checksum alg desc and value (or value's room) */
|
||||||
|
numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= "
|
||||||
|
+ numberOfEventTypes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** MySQL 5.0 format descriptions. */
|
||||||
|
public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_5_x = new FormatDescriptionLogEvent(4);
|
||||||
|
|
||||||
|
/** MySQL 4.0.x (x>=2) format descriptions. */
|
||||||
|
public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_4_0_x = new FormatDescriptionLogEvent(3);
|
||||||
|
|
||||||
|
/** MySQL 3.23 format descriptions. */
|
||||||
|
public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_3_23 = new FormatDescriptionLogEvent(1);
|
||||||
|
|
||||||
|
public static FormatDescriptionLogEvent getFormatDescription(final int binlogVersion) throws IOException {
|
||||||
|
/* identify binlog format */
|
||||||
|
switch (binlogVersion) {
|
||||||
|
case 4: /* MySQL 5.0 */
|
||||||
|
return FORMAT_DESCRIPTION_EVENT_5_x;
|
||||||
|
case 3:
|
||||||
|
return FORMAT_DESCRIPTION_EVENT_4_0_x;
|
||||||
|
case 1:
|
||||||
|
return FORMAT_DESCRIPTION_EVENT_3_23;
|
||||||
|
default:
|
||||||
|
throw new IOException("Unknown binlog version: " + binlogVersion);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public FormatDescriptionLogEvent(final int binlogVersion){
|
||||||
|
this.binlogVersion = binlogVersion;
|
||||||
|
|
||||||
|
postHeaderLen = new short[ENUM_END_EVENT];
|
||||||
|
/* identify binlog format */
|
||||||
|
switch (binlogVersion) {
|
||||||
|
case 4: /* MySQL 5.0 */
|
||||||
|
serverVersion = SERVER_VERSION;
|
||||||
|
commonHeaderLen = LOG_EVENT_HEADER_LEN;
|
||||||
|
numberOfEventTypes = LOG_EVENT_TYPES;
|
||||||
|
|
||||||
|
/* Note: all event types must explicitly fill in their lengths here. */
|
||||||
|
postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
|
||||||
|
postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
|
||||||
|
postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
|
||||||
|
postHeaderLen[ROTATE_EVENT - 1] = ROTATE_HEADER_LEN;
|
||||||
|
postHeaderLen[INTVAR_EVENT - 1] = INTVAR_HEADER_LEN;
|
||||||
|
postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[SLAVE_EVENT - 1] = SLAVE_HEADER_LEN;
|
||||||
|
postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN;
|
||||||
|
postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN;
|
||||||
|
postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN;
|
||||||
|
postHeaderLen[NEW_LOAD_EVENT - 1] = NEW_LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[RAND_EVENT - 1] = RAND_HEADER_LEN;
|
||||||
|
postHeaderLen[USER_VAR_EVENT - 1] = USER_VAR_HEADER_LEN;
|
||||||
|
postHeaderLen[FORMAT_DESCRIPTION_EVENT - 1] = FORMAT_DESCRIPTION_HEADER_LEN;
|
||||||
|
postHeaderLen[XID_EVENT - 1] = XID_HEADER_LEN;
|
||||||
|
postHeaderLen[BEGIN_LOAD_QUERY_EVENT - 1] = BEGIN_LOAD_QUERY_HEADER_LEN;
|
||||||
|
postHeaderLen[EXECUTE_LOAD_QUERY_EVENT - 1] = EXECUTE_LOAD_QUERY_HEADER_LEN;
|
||||||
|
postHeaderLen[TABLE_MAP_EVENT - 1] = TABLE_MAP_HEADER_LEN;
|
||||||
|
postHeaderLen[WRITE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
|
||||||
|
postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
|
||||||
|
postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
|
||||||
|
/*
|
||||||
|
* We here have the possibility to simulate a master of before we changed the table map id to be stored
|
||||||
|
* in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
|
||||||
|
* compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
|
||||||
|
* that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
|
||||||
|
* test (rpl_row_4_bytes) too.
|
||||||
|
*/
|
||||||
|
postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
|
||||||
|
postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
|
||||||
|
postHeaderLen[ROWS_QUERY_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
|
||||||
|
postHeaderLen[WRITE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
|
||||||
|
postHeaderLen[UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
|
||||||
|
postHeaderLen[DELETE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
|
||||||
|
postHeaderLen[GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
|
||||||
|
postHeaderLen[ANONYMOUS_GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
|
||||||
|
postHeaderLen[PREVIOUS_GTIDS_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
|
||||||
|
// mariadb 10
|
||||||
|
postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;
|
||||||
|
postHeaderLen[BINLOG_CHECKPOINT_EVENT - 1] = BINLOG_CHECKPOINT_HEADER_LEN;
|
||||||
|
postHeaderLen[GTID_EVENT - 1] = GTID_HEADER_LEN;
|
||||||
|
postHeaderLen[GTID_LIST_EVENT - 1] = GTID_LIST_HEADER_LEN;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 3: /* 4.0.x x>=2 */
|
||||||
|
/*
|
||||||
|
* We build an artificial (i.e. not sent by the master) event, which describes what those old master
|
||||||
|
* versions send.
|
||||||
|
*/
|
||||||
|
serverVersion = "4.0";
|
||||||
|
commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
|
||||||
|
* in older versions. We use the events known by version 3, even if version 1 had only a subset of them
|
||||||
|
* (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
|
||||||
|
* detect less corruptions).
|
||||||
|
*/
|
||||||
|
numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
|
||||||
|
|
||||||
|
postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
|
||||||
|
postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_MINIMAL_LEN;
|
||||||
|
postHeaderLen[ROTATE_EVENT - 1] = ROTATE_HEADER_LEN;
|
||||||
|
postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN;
|
||||||
|
postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN;
|
||||||
|
postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN;
|
||||||
|
postHeaderLen[NEW_LOAD_EVENT - 1] = postHeaderLen[LOAD_EVENT - 1];
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 1: /* 3.23 */
|
||||||
|
/*
|
||||||
|
* We build an artificial (i.e. not sent by the master) event, which describes what those old master
|
||||||
|
* versions send.
|
||||||
|
*/
|
||||||
|
serverVersion = "3.23";
|
||||||
|
commonHeaderLen = OLD_HEADER_LEN;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
|
||||||
|
* in older versions. We use the events known by version 3, even if version 1 had only a subset of them
|
||||||
|
* (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
|
||||||
|
* detect less corruptions).
|
||||||
|
*/
|
||||||
|
numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
|
||||||
|
|
||||||
|
postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
|
||||||
|
postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_MINIMAL_LEN;
|
||||||
|
postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN;
|
||||||
|
postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN;
|
||||||
|
postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN;
|
||||||
|
postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN;
|
||||||
|
postHeaderLen[NEW_LOAD_EVENT - 1] = postHeaderLen[LOAD_EVENT - 1];
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
numberOfEventTypes = 0;
|
||||||
|
commonHeaderLen = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void calcServerVersionSplit() {
|
||||||
|
doServerVersionSplit(serverVersion, serverVersionSplit);
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getVersionProduct() {
|
||||||
|
return versionProduct(serverVersionSplit);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isVersionBeforeChecksum() {
|
||||||
|
return getVersionProduct() < checksumVersionProduct;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void doServerVersionSplit(String serverVersion, int[] versionSplit) {
|
||||||
|
String[] split = serverVersion.split("\\.");
|
||||||
|
if (split.length < 3) {
|
||||||
|
versionSplit[0] = 0;
|
||||||
|
versionSplit[1] = 0;
|
||||||
|
versionSplit[2] = 0;
|
||||||
|
} else {
|
||||||
|
int j = 0;
|
||||||
|
for (int i = 0; i <= 2; i++) {
|
||||||
|
String str = split[i];
|
||||||
|
for (j = 0; j < str.length(); j++) {
|
||||||
|
if (Character.isDigit(str.charAt(j)) == false) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (j > 0) {
|
||||||
|
versionSplit[i] = Integer.valueOf(str.substring(0, j), 10);
|
||||||
|
} else {
|
||||||
|
versionSplit[0] = 0;
|
||||||
|
versionSplit[1] = 0;
|
||||||
|
versionSplit[2] = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static long versionProduct(int[] versionSplit) {
|
||||||
|
return ((versionSplit[0] * 256 + versionSplit[1]) * 256 + versionSplit[2]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getCommonHeaderLen() {
|
||||||
|
return commonHeaderLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final short[] getPostHeaderLen() {
|
||||||
|
return postHeaderLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author jianghang 2013-4-8 上午12:36:29
|
||||||
|
* @version 1.0.3
|
||||||
|
* @since mysql 5.6 / mariadb10
|
||||||
|
*/
|
||||||
|
public class GtidLogEvent extends LogEvent {
|
||||||
|
|
||||||
|
// / Length of the commit_flag in event encoding
|
||||||
|
public static final int ENCODED_FLAG_LENGTH = 1;
|
||||||
|
// / Length of SID in event encoding
|
||||||
|
public static final int ENCODED_SID_LENGTH = 16;
|
||||||
|
|
||||||
|
private boolean commitFlag;
|
||||||
|
|
||||||
|
public GtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
// final int postHeaderLen = descriptionEvent.postHeaderLen[header.type
|
||||||
|
// - 1];
|
||||||
|
|
||||||
|
buffer.position(commonHeaderLen);
|
||||||
|
commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH
|
||||||
|
|
||||||
|
// ignore gtid info read
|
||||||
|
// sid.copy_from((uchar *)ptr_buffer);
|
||||||
|
// ptr_buffer+= ENCODED_SID_LENGTH;
|
||||||
|
//
|
||||||
|
// // SIDNO is only generated if needed, in get_sidno().
|
||||||
|
// spec.gtid.sidno= -1;
|
||||||
|
//
|
||||||
|
// spec.gtid.gno= uint8korr(ptr_buffer);
|
||||||
|
// ptr_buffer+= ENCODED_GNO_LENGTH;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isCommitFlag() {
|
||||||
|
return commitFlag;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
* Replication event to ensure to slave that master is alive.
|
||||||
|
* The event is originated by master's dump thread and sent straight to
|
||||||
|
* slave without being logged. Slave itself does not store it in relay log
|
||||||
|
* but rather uses a data for immediate checks and throws away the event.
|
||||||
|
*
|
||||||
|
* Two members of the class log_ident and Log_event::log_pos comprise
|
||||||
|
* @see the event_coordinates instance. The coordinates that a heartbeat
|
||||||
|
* instance carries correspond to the last event master has sent from
|
||||||
|
* its binlog.
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* @author jianghang 2013-4-8 上午12:36:29
|
||||||
|
* @version 1.0.3
|
||||||
|
* @since mysql 5.6
|
||||||
|
*/
|
||||||
|
public class HeartbeatLogEvent extends LogEvent {
|
||||||
|
|
||||||
|
public static final int FN_REFLEN = 512; /* Max length of full path-name */
|
||||||
|
private int identLen;
|
||||||
|
private String logIdent;
|
||||||
|
|
||||||
|
public HeartbeatLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
identLen = buffer.limit() - commonHeaderLen;
|
||||||
|
if (identLen > FN_REFLEN - 1) {
|
||||||
|
identLen = FN_REFLEN - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
logIdent = buffer.getFullString(commonHeaderLen, identLen, LogBuffer.ISO_8859_1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getIdentLen() {
|
||||||
|
return identLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLogIdent() {
|
||||||
|
return logIdent;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
* Base class for ignorable log events. Events deriving from
|
||||||
|
this class can be safely ignored by slaves that cannot
|
||||||
|
recognize them. Newer slaves, will be able to read and
|
||||||
|
handle them. This has been designed to be an open-ended
|
||||||
|
architecture, so adding new derived events shall not harm
|
||||||
|
the old slaves that support ignorable log event mechanism
|
||||||
|
(they will just ignore unrecognized ignorable events).
|
||||||
|
|
||||||
|
@note The only thing that makes an event ignorable is that it has
|
||||||
|
the LOG_EVENT_IGNORABLE_F flag set. It is not strictly necessary
|
||||||
|
that ignorable event types derive from Ignorable_log_event; they may
|
||||||
|
just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
|
||||||
|
argument to the Log_event constructor.
|
||||||
|
</pre>
|
||||||
|
*
|
||||||
|
* @author jianghang 2013-4-8 上午12:36:29
|
||||||
|
* @version 1.0.3
|
||||||
|
* @since mysql 5.6
|
||||||
|
*/
|
||||||
|
public class IgnorableLogEvent extends LogEvent {
|
||||||
|
|
||||||
|
public IgnorableLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
// do nothing , just ignore log event
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Class representing an incident, an occurance out of the ordinary, that
|
||||||
|
* happened on the master.
|
||||||
|
*
|
||||||
|
* The event is used to inform the slave that something out of the ordinary
|
||||||
|
* happened on the master that might cause the database to be in an inconsistent
|
||||||
|
* state.
|
||||||
|
*
|
||||||
|
* <table id="IncidentFormat">
|
||||||
|
* <caption>Incident event format</caption>
|
||||||
|
* <tr>
|
||||||
|
* <th>Symbol</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
* <tr>
|
||||||
|
* <td>INCIDENT</td>
|
||||||
|
* <td align="right">2</td>
|
||||||
|
* <td>Incident number as an unsigned integer</td>
|
||||||
|
* </tr>
|
||||||
|
* <tr>
|
||||||
|
* <td>MSGLEN</td>
|
||||||
|
* <td align="right">1</td>
|
||||||
|
* <td>Message length as an unsigned integer</td>
|
||||||
|
* </tr>
|
||||||
|
* <tr>
|
||||||
|
* <td>MESSAGE</td>
|
||||||
|
* <td align="right">MSGLEN</td>
|
||||||
|
* <td>The message, if present. Not null terminated.</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class IncidentLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
public static final int INCIDENT_NONE = 0;
|
||||||
|
|
||||||
|
/** There are possibly lost events in the replication stream */
|
||||||
|
public static final int INCIDENT_LOST_EVENTS = 1;
|
||||||
|
|
||||||
|
/** Shall be last event of the enumeration */
|
||||||
|
public static final int INCIDENT_COUNT = 2;
|
||||||
|
|
||||||
|
private final int incident;
|
||||||
|
private final String message;
|
||||||
|
|
||||||
|
public IncidentLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
|
||||||
|
|
||||||
|
buffer.position(commonHeaderLen);
|
||||||
|
final int incidentNumber = buffer.getUint16();
|
||||||
|
if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
|
||||||
|
{
|
||||||
|
// If the incident is not recognized, this binlog event is
|
||||||
|
// invalid. If we set incident_number to INCIDENT_NONE, the
|
||||||
|
// invalidity will be detected by is_valid().
|
||||||
|
incident = INCIDENT_NONE;
|
||||||
|
message = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
incident = incidentNumber;
|
||||||
|
|
||||||
|
buffer.position(commonHeaderLen + postHeaderLen);
|
||||||
|
message = buffer.getString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getIncident()
|
||||||
|
{
|
||||||
|
return incident;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getMessage()
|
||||||
|
{
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An Intvar_log_event will be created just before a Query_log_event, if the
|
||||||
|
* query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
|
||||||
|
* Intvar_log_event holds the value of one of these variables.
|
||||||
|
*
|
||||||
|
* Binary Format
|
||||||
|
*
|
||||||
|
* The Post-Header for this event type is empty. The Body has two components:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Body for Intvar_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>type</td>
|
||||||
|
* <td>1 byte enumeration</td>
|
||||||
|
* <td>One byte identifying the type of variable stored. Currently, two
|
||||||
|
* identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>value</td>
|
||||||
|
* <td>8 byte unsigned integer</td>
|
||||||
|
* <td>The value of the variable.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class IntvarLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Fixed data part: Empty
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Variable data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
|
||||||
|
* 1 or INSERT_ID_EVENT = 2.</li>
|
||||||
|
* <li>8 bytes. An unsigned integer indicating the value to be used for the
|
||||||
|
* LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
private final long value;
|
||||||
|
private final int type;
|
||||||
|
|
||||||
|
/* Intvar event data */
|
||||||
|
public static final int I_TYPE_OFFSET = 0;
|
||||||
|
public static final int I_VAL_OFFSET = 1;
|
||||||
|
|
||||||
|
// enum Int_event_type
|
||||||
|
public static final int INVALID_INT_EVENT = 0;
|
||||||
|
public static final int LAST_INSERT_ID_EVENT = 1;
|
||||||
|
public static final int INSERT_ID_EVENT = 2;
|
||||||
|
|
||||||
|
public IntvarLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
/* The Post-Header is empty. The Varible Data part begins immediately. */
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen
|
||||||
|
+ descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
|
||||||
|
+ I_TYPE_OFFSET);
|
||||||
|
type = buffer.getInt8(); // I_TYPE_OFFSET
|
||||||
|
value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getType()
|
||||||
|
{
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getValue()
|
||||||
|
{
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getQuery()
|
||||||
|
{
|
||||||
|
return "SET INSERT_ID = " + value;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This log event corresponds to a "LOAD DATA INFILE" SQL query on the following
|
||||||
|
* form:
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* (1) USE db;
|
||||||
|
* (2) LOAD DATA [CONCURRENT] [LOCAL] INFILE 'file_name'
|
||||||
|
* (3) [REPLACE | IGNORE]
|
||||||
|
* (4) INTO TABLE 'table_name'
|
||||||
|
* (5) [FIELDS
|
||||||
|
* (6) [TERMINATED BY 'field_term']
|
||||||
|
* (7) [[OPTIONALLY] ENCLOSED BY 'enclosed']
|
||||||
|
* (8) [ESCAPED BY 'escaped']
|
||||||
|
* (9) ]
|
||||||
|
* (10) [LINES
|
||||||
|
* (11) [TERMINATED BY 'line_term']
|
||||||
|
* (12) [LINES STARTING BY 'line_start']
|
||||||
|
* (13) ]
|
||||||
|
* (14) [IGNORE skip_lines LINES]
|
||||||
|
* (15) (field_1, field_2, ..., field_n)
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* Binary Format: The Post-Header consists of the following six components.
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Post-Header for Load_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>slave_proxy_id</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>An integer identifying the client thread that issued the query. The id is
|
||||||
|
* unique per server. (Note, however, that two threads on different servers may
|
||||||
|
* have the same slave_proxy_id.) This is used when a client thread creates a
|
||||||
|
* temporary table local to the client. The slave_proxy_id is used to
|
||||||
|
* distinguish temporary tables that belong to different clients.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>exec_time</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The time from when the query started to when it was logged in the binlog,
|
||||||
|
* in seconds.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>skip_lines</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The number on line (14) above, if present, or 0 if line (14) is left out.
|
||||||
|
* </td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>table_name_len</td>
|
||||||
|
* <td>1 byte unsigned integer</td>
|
||||||
|
* <td>The length of 'table_name' on line (4) above.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>db_len</td>
|
||||||
|
* <td>1 byte unsigned integer</td>
|
||||||
|
* <td>The length of 'db' on line (1) above.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>num_fields</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The number n of fields on line (15) above.</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* The Body contains the following components.
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Body of Load_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>sql_ex</td>
|
||||||
|
* <td>variable length</td>
|
||||||
|
*
|
||||||
|
* <td>Describes the part of the query on lines (3) and (5)–(13) above.
|
||||||
|
* More precisely, it stores the five strings (on lines) field_term (6),
|
||||||
|
* enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
|
||||||
|
* bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
|
||||||
|
* OPTIONALLY (7).
|
||||||
|
*
|
||||||
|
* The data is stored in one of two formats, called "old" and "new". The type
|
||||||
|
* field of Common-Header determines which of these two formats is used: type
|
||||||
|
* LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
|
||||||
|
* that the new format is used. When MySQL writes a Load_log_event, it uses the
|
||||||
|
* new format if at least one of the five strings is two or more bytes long.
|
||||||
|
* Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
|
||||||
|
* used.
|
||||||
|
*
|
||||||
|
* The new and old format differ in the way the five strings are stored.
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>In the new format, the strings are stored in the order field_term,
|
||||||
|
* enclosed, escaped, line_term, line_start. Each string consists of a length (1
|
||||||
|
* byte), followed by a sequence of characters (0-255 bytes). Finally, a boolean
|
||||||
|
* combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
|
||||||
|
* IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
|
||||||
|
* the presence of the corresponding keyword in the SQL query.
|
||||||
|
*
|
||||||
|
* <li>In the old format, we know that each string has length 0 or 1. Therefore,
|
||||||
|
* only the first byte of each string is stored. The order of the strings is the
|
||||||
|
* same as in the new format. These five bytes are followed by the same 1 byte
|
||||||
|
* bitfield as in the new format. Finally, a 1 byte bitfield called empty_flags
|
||||||
|
* is stored. The low 5 bits of empty_flags indicate which of the five strings
|
||||||
|
* have length 0. For each of the following flags that is set, the corresponding
|
||||||
|
* string has length 0; for the flags that are not set, the string has length 1:
|
||||||
|
* FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
|
||||||
|
* LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Thus, the size of the new format is 6 bytes + the sum of the sizes of the
|
||||||
|
* five strings. The size of the old format is always 7 bytes.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>field_lens</td>
|
||||||
|
* <td>num_fields 1 byte unsigned integers</td>
|
||||||
|
* <td>An array of num_fields integers representing the length of each field in
|
||||||
|
* the query. (num_fields is from the Post-Header).</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>fields</td>
|
||||||
|
* <td>num_fields null-terminated strings</td>
|
||||||
|
* <td>An array of num_fields null-terminated strings, each representing a field
|
||||||
|
* in the query. (The trailing zero is redundant, since the length are stored in
|
||||||
|
* the num_fields array.) The total length of all strings equals to the sum of
|
||||||
|
* all field_lens, plus num_fields bytes for all the trailing zeros.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>table_name</td>
|
||||||
|
* <td>null-terminated string of length table_len+1 bytes</td>
|
||||||
|
* <td>The 'table_name' from the query, as a null-terminated string. (The
|
||||||
|
* trailing zero is actually redundant since the table_len is known from
|
||||||
|
* Post-Header.)</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>db</td>
|
||||||
|
* <td>null-terminated string of length db_len+1 bytes</td>
|
||||||
|
* <td>The 'db' from the query, as a null-terminated string. (The trailing zero
|
||||||
|
* is actually redundant since the db_len is known from Post-Header.)</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>file_name</td>
|
||||||
|
* <td>variable length string without trailing zero, extending to the end of the
|
||||||
|
* event (determined by the length field of the Common-Header)</td>
|
||||||
|
* <td>The 'file_name' from the query.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* This event type is understood by current versions, but only generated by
|
||||||
|
* MySQL 3.23 and earlier.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public class LoadLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
private String table;
|
||||||
|
private String db;
|
||||||
|
private String fname;
|
||||||
|
private int skipLines;
|
||||||
|
private int numFields;
|
||||||
|
private String[] fields;
|
||||||
|
|
||||||
|
/* sql_ex_info */
|
||||||
|
private String fieldTerm;
|
||||||
|
private String lineTerm;
|
||||||
|
private String lineStart;
|
||||||
|
private String enclosed;
|
||||||
|
private String escaped;
|
||||||
|
private int optFlags;
|
||||||
|
private int emptyFlags;
|
||||||
|
|
||||||
|
private long execTime;
|
||||||
|
|
||||||
|
/* Load event post-header */
|
||||||
|
public static final int L_THREAD_ID_OFFSET = 0;
|
||||||
|
public static final int L_EXEC_TIME_OFFSET = 4;
|
||||||
|
public static final int L_SKIP_LINES_OFFSET = 8;
|
||||||
|
public static final int L_TBL_LEN_OFFSET = 12;
|
||||||
|
public static final int L_DB_LEN_OFFSET = 13;
|
||||||
|
public static final int L_NUM_FIELDS_OFFSET = 14;
|
||||||
|
public static final int L_SQL_EX_OFFSET = 18;
|
||||||
|
public static final int L_DATA_OFFSET = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
|
||||||
|
|
||||||
|
/*
|
||||||
|
These are flags and structs to handle all the LOAD DATA INFILE options (LINES
|
||||||
|
TERMINATED etc).
|
||||||
|
DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
|
||||||
|
DATA).
|
||||||
|
*/
|
||||||
|
public static final int DUMPFILE_FLAG = 0x1;
|
||||||
|
public static final int OPT_ENCLOSED_FLAG = 0x2;
|
||||||
|
public static final int REPLACE_FLAG = 0x4;
|
||||||
|
public static final int IGNORE_FLAG = 0x8;
|
||||||
|
|
||||||
|
public static final int FIELD_TERM_EMPTY = 0x1;
|
||||||
|
public static final int ENCLOSED_EMPTY = 0x2;
|
||||||
|
public static final int LINE_TERM_EMPTY = 0x4;
|
||||||
|
public static final int LINE_START_EMPTY = 0x8;
|
||||||
|
public static final int ESCAPED_EMPTY = 0x10;
|
||||||
|
|
||||||
|
public LoadLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
|
||||||
|
/*
|
||||||
|
* I (Guilhem) manually tested replication of LOAD DATA INFILE for
|
||||||
|
* 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
|
||||||
|
*/
|
||||||
|
copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
|
||||||
|
+ descriptionEvent.commonHeaderLen : loadHeaderLen
|
||||||
|
+ FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
|
||||||
|
descriptionEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
|
||||||
|
*/
|
||||||
|
protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
/* this is the beginning of the post-header */
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
|
||||||
|
|
||||||
|
execTime = buffer.getUint32(); // L_EXEC_TIME_OFFSET
|
||||||
|
skipLines = (int) buffer.getUint32(); // L_SKIP_LINES_OFFSET
|
||||||
|
final int tableNameLen = buffer.getUint8(); // L_TBL_LEN_OFFSET
|
||||||
|
final int dbLen = buffer.getUint8(); // L_DB_LEN_OFFSET
|
||||||
|
numFields = (int) buffer.getUint32(); // L_NUM_FIELDS_OFFSET
|
||||||
|
|
||||||
|
buffer.position(bodyOffset);
|
||||||
|
/*
|
||||||
|
* Sql_ex.init() on success returns the pointer to the first byte after
|
||||||
|
* the sql_ex structure, which is the start of field lengths array.
|
||||||
|
*/
|
||||||
|
if (header.type != LOAD_EVENT /* use_new_format */)
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* The code below assumes that buf will not disappear from under our
|
||||||
|
* feet during the lifetime of the event. This assumption holds true
|
||||||
|
* in the slave thread if the log is in new format, but is not the
|
||||||
|
* case when we have old format because we will be reusing net
|
||||||
|
* buffer to read the actual file before we write out the
|
||||||
|
* Create_file event.
|
||||||
|
*/
|
||||||
|
fieldTerm = buffer.getString();
|
||||||
|
enclosed = buffer.getString();
|
||||||
|
lineTerm = buffer.getString();
|
||||||
|
lineStart = buffer.getString();
|
||||||
|
escaped = buffer.getString();
|
||||||
|
optFlags = buffer.getInt8();
|
||||||
|
emptyFlags = 0;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
fieldTerm = buffer.getFixString(1);
|
||||||
|
enclosed = buffer.getFixString(1);
|
||||||
|
lineTerm = buffer.getFixString(1);
|
||||||
|
lineStart = buffer.getFixString(1);
|
||||||
|
escaped = buffer.getFixString(1);
|
||||||
|
optFlags = buffer.getUint8();
|
||||||
|
emptyFlags = buffer.getUint8();
|
||||||
|
|
||||||
|
if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
|
||||||
|
fieldTerm = null;
|
||||||
|
if ((emptyFlags & ENCLOSED_EMPTY) != 0)
|
||||||
|
enclosed = null;
|
||||||
|
if ((emptyFlags & LINE_TERM_EMPTY) != 0)
|
||||||
|
lineTerm = null;
|
||||||
|
if ((emptyFlags & LINE_START_EMPTY) != 0)
|
||||||
|
lineStart = null;
|
||||||
|
if ((emptyFlags & ESCAPED_EMPTY) != 0)
|
||||||
|
escaped = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
final int fieldLenPos = buffer.position();
|
||||||
|
buffer.forward(numFields);
|
||||||
|
fields = new String[numFields];
|
||||||
|
for (int i = 0; i < numFields; i++)
|
||||||
|
{
|
||||||
|
final int fieldLen = buffer.getUint8(fieldLenPos + i);
|
||||||
|
fields[i] = buffer.getFixString(fieldLen + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
table = buffer.getFixString(tableNameLen + 1);
|
||||||
|
db = buffer.getFixString(dbLen + 1);
|
||||||
|
|
||||||
|
// null termination is accomplished by the caller
|
||||||
|
final int from = buffer.position();
|
||||||
|
final int end = from + buffer.limit();
|
||||||
|
int found = from;
|
||||||
|
for (; (found < end) && buffer.getInt8(found) != '\0'; found++)
|
||||||
|
/* empty loop */;
|
||||||
|
fname = buffer.getString(found);
|
||||||
|
buffer.forward(1); // The + 1 is for \0 terminating fname
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getTable()
|
||||||
|
{
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getDb()
|
||||||
|
{
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getFname()
|
||||||
|
{
|
||||||
|
return fname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getSkipLines()
|
||||||
|
{
|
||||||
|
return skipLines;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String[] getFields()
|
||||||
|
{
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getFieldTerm()
|
||||||
|
{
|
||||||
|
return fieldTerm;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getLineTerm()
|
||||||
|
{
|
||||||
|
return lineTerm;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getLineStart()
|
||||||
|
{
|
||||||
|
return lineStart;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getEnclosed()
|
||||||
|
{
|
||||||
|
return enclosed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getEscaped()
|
||||||
|
{
|
||||||
|
return escaped;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getOptFlags()
|
||||||
|
{
|
||||||
|
return optFlags;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getEmptyFlags()
|
||||||
|
{
|
||||||
|
return emptyFlags;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getExecTime()
|
||||||
|
{
|
||||||
|
return execTime;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,310 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Common-Header, documented in the table @ref Table_common_header "below",
|
||||||
|
* always has the same form and length within one version of MySQL. Each event
|
||||||
|
* type specifies a format and length of the Post-Header. The length of the
|
||||||
|
* Common-Header is the same for all events of the same type. The Body may be of
|
||||||
|
* different format and length even for different events of the same type. The
|
||||||
|
* binary formats of Post-Header and Body are documented separately in each
|
||||||
|
* subclass. The binary format of Common-Header is as follows.
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Common-Header</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>timestamp</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The time when the query started, in seconds since 1970.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>type</td>
|
||||||
|
* <td>1 byte enumeration</td>
|
||||||
|
* <td>See enum #Log_event_type.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>server_id</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>Server ID of the server that created the event.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>total_size</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The total size of this event, in bytes. In other words, this is the sum
|
||||||
|
* of the sizes of Common-Header, Post-Header, and Body.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>master_position</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The position of the next event in the master binary log, in bytes from
|
||||||
|
* the beginning of the file. In a binlog that is not a relay log, this is just
|
||||||
|
* the position of the next event, in bytes from the beginning of the file. In a
|
||||||
|
* relay log, this is the position of the next event in the master's binlog.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>flags</td>
|
||||||
|
* <td>2 byte bitfield</td>
|
||||||
|
* <td>See Log_event::flags.</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* Summing up the numbers above, we see that the total size of the common header
|
||||||
|
* is 19 bytes.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class LogHeader
|
||||||
|
{
|
||||||
|
protected final int type;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The offset in the log where this event originally appeared (it is
|
||||||
|
* preserved in relay logs, making SHOW SLAVE STATUS able to print
|
||||||
|
* coordinates of the event in the master's binlog). Note: when a
|
||||||
|
* transaction is written by the master to its binlog (wrapped in
|
||||||
|
* BEGIN/COMMIT) the log_pos of all the queries it contains is the one of
|
||||||
|
* the BEGIN (this way, when one does SHOW SLAVE STATUS it sees the offset
|
||||||
|
* of the BEGIN, which is logical as rollback may occur), except the COMMIT
|
||||||
|
* query which has its real offset.
|
||||||
|
*/
|
||||||
|
protected long logPos;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Timestamp on the master(for debugging and replication of
|
||||||
|
* NOW()/TIMESTAMP). It is important for queries and LOAD DATA INFILE. This
|
||||||
|
* is set at the event's creation time, except for Query and Load (et al.)
|
||||||
|
* events where this is set at the query's execution time, which guarantees
|
||||||
|
* good replication (otherwise, we could have a query and its event with
|
||||||
|
* different timestamps).
|
||||||
|
*/
|
||||||
|
protected long when;
|
||||||
|
|
||||||
|
/** Number of bytes written by write() function */
|
||||||
|
protected int eventLen;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The master's server id (is preserved in the relay log; used to prevent
|
||||||
|
* from infinite loops in circular replication).
|
||||||
|
*/
|
||||||
|
protected long serverId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Some 16 flags. See the definitions above for LOG_EVENT_TIME_F,
|
||||||
|
* LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
|
||||||
|
* LOG_EVENT_SUPPRESS_USE_F for notes.
|
||||||
|
*/
|
||||||
|
protected int flags;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The value is set by caller of FD constructor and
|
||||||
|
* Log_event::write_header() for the rest.
|
||||||
|
* In the FD case it's propagated into the last byte
|
||||||
|
* of post_header_len[] at FD::write().
|
||||||
|
* On the slave side the value is assigned from post_header_len[last]
|
||||||
|
* of the last seen FD event.
|
||||||
|
*/
|
||||||
|
protected int checksumAlg;
|
||||||
|
/**
|
||||||
|
Placeholder for event checksum while writing to binlog.
|
||||||
|
*/
|
||||||
|
protected long crc; // ha_checksum
|
||||||
|
|
||||||
|
/* for Start_event_v3 */
|
||||||
|
public LogHeader(final int type)
|
||||||
|
{
|
||||||
|
this.type = type;
|
||||||
|
}
|
||||||
|
|
||||||
|
public LogHeader(LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
when = buffer.getUint32();
|
||||||
|
type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET;
|
||||||
|
serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET;
|
||||||
|
eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET;
|
||||||
|
|
||||||
|
if (descriptionEvent.binlogVersion == 1)
|
||||||
|
{
|
||||||
|
logPos = 0;
|
||||||
|
flags = 0;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* 4.0 or newer */
|
||||||
|
logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET
|
||||||
|
/*
|
||||||
|
If the log is 4.0 (so here it can only be a 4.0 relay log read by
|
||||||
|
the SQL thread or a 4.0 master binlog read by the I/O thread),
|
||||||
|
log_pos is the beginning of the event: we transform it into the end
|
||||||
|
of the event, which is more useful.
|
||||||
|
But how do you know that the log is 4.0: you know it if
|
||||||
|
description_event is version 3 *and* you are not reading a
|
||||||
|
Format_desc (remember that mysqlbinlog starts by assuming that 5.0
|
||||||
|
logs are in 4.0 format, until it finds a Format_desc).
|
||||||
|
*/
|
||||||
|
if (descriptionEvent.binlogVersion == 3
|
||||||
|
&& type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0)
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
If log_pos=0, don't change it. log_pos==0 is a marker to mean
|
||||||
|
"don't change rli->group_master_log_pos" (see
|
||||||
|
inc_group_relay_log_pos()). As it is unreal log_pos, adding the
|
||||||
|
event len's is nonsense. For example, a fake Rotate event should
|
||||||
|
not have its log_pos (which is 0) changed or it will modify
|
||||||
|
Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
|
||||||
|
value of (a non-zero offset which does not exist in the master's
|
||||||
|
binlog, so which will cause problems if the user uses this value
|
||||||
|
in CHANGE MASTER).
|
||||||
|
*/
|
||||||
|
logPos += eventLen; /* purecov: inspected */
|
||||||
|
}
|
||||||
|
|
||||||
|
flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET
|
||||||
|
if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT)
|
||||||
|
|| (type == LogEvent.ROTATE_EVENT))
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
These events always have a header which stops here (i.e. their
|
||||||
|
header is FROZEN).
|
||||||
|
*/
|
||||||
|
/*
|
||||||
|
Initialization to zero of all other Log_event members as they're
|
||||||
|
not specified. Currently there are no such members; in the future
|
||||||
|
there will be an event UID (but Format_description and Rotate
|
||||||
|
don't need this UID, as they are not propagated through
|
||||||
|
--log-slave-updates (remember the UID is used to not play a query
|
||||||
|
twice when you have two masters which are slaves of a 3rd master).
|
||||||
|
Then we are done.
|
||||||
|
*/
|
||||||
|
|
||||||
|
if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) {
|
||||||
|
int commonHeaderLen = buffer.getUint8(FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN
|
||||||
|
+ FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET);
|
||||||
|
buffer.position(commonHeaderLen + FormatDescriptionLogEvent.ST_SERVER_VER_OFFSET);
|
||||||
|
String serverVersion = buffer.getFixString(FormatDescriptionLogEvent.ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
|
||||||
|
int versionSplit[] = new int[] { 0, 0, 0 };
|
||||||
|
FormatDescriptionLogEvent.doServerVersionSplit(serverVersion, versionSplit);
|
||||||
|
checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
|
||||||
|
if (FormatDescriptionLogEvent.versionProduct(versionSplit) >= FormatDescriptionLogEvent.checksumVersionProduct) {
|
||||||
|
buffer.position(eventLen - LogEvent.BINLOG_CHECKSUM_LEN - LogEvent.BINLOG_CHECKSUM_ALG_DESC_LEN);
|
||||||
|
checksumAlg = buffer.getUint8();
|
||||||
|
}
|
||||||
|
|
||||||
|
processCheckSum(buffer);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
CRC verification by SQL and Show-Binlog-Events master side.
|
||||||
|
The caller has to provide @description_event->checksum_alg to
|
||||||
|
be the last seen FD's (A) descriptor.
|
||||||
|
If event is FD the descriptor is in it.
|
||||||
|
Notice, FD of the binlog can be only in one instance and therefore
|
||||||
|
Show-Binlog-Events executing master side thread needs just to know
|
||||||
|
the only FD's (A) value - whereas RL can contain more.
|
||||||
|
In the RL case, the alg is kept in FD_e (@description_event) which is reset
|
||||||
|
to the newer read-out event after its execution with possibly new alg descriptor.
|
||||||
|
Therefore in a typical sequence of RL:
|
||||||
|
{FD_s^0, FD_m, E_m^1} E_m^1
|
||||||
|
will be verified with (A) of FD_m.
|
||||||
|
|
||||||
|
See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs
|
||||||
|
lines (log.h).
|
||||||
|
|
||||||
|
Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF.
|
||||||
|
*/
|
||||||
|
checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg
|
||||||
|
processCheckSum(buffer);
|
||||||
|
/* otherwise, go on with reading the header from buf (nothing now) */
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The different types of log events.
|
||||||
|
*/
|
||||||
|
public final int getType()
|
||||||
|
{
|
||||||
|
return type;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The position of the next event in the master binary log, in bytes from
|
||||||
|
* the beginning of the file. In a binlog that is not a relay log, this is
|
||||||
|
* just the position of the next event, in bytes from the beginning of the
|
||||||
|
* file. In a relay log, this is the position of the next event in the
|
||||||
|
* master's binlog.
|
||||||
|
*/
|
||||||
|
public final long getLogPos()
|
||||||
|
{
|
||||||
|
return logPos;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The total size of this event, in bytes. In other words, this is the sum
|
||||||
|
* of the sizes of Common-Header, Post-Header, and Body.
|
||||||
|
*/
|
||||||
|
public final int getEventLen()
|
||||||
|
{
|
||||||
|
return eventLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The time when the query started, in seconds since 1970.
|
||||||
|
*/
|
||||||
|
public final long getWhen()
|
||||||
|
{
|
||||||
|
return when;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Server ID of the server that created the event.
|
||||||
|
*/
|
||||||
|
public final long getServerId()
|
||||||
|
{
|
||||||
|
return serverId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Some 16 flags. See the definitions above for LOG_EVENT_TIME_F,
|
||||||
|
* LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
|
||||||
|
* LOG_EVENT_SUPPRESS_USE_F for notes.
|
||||||
|
*/
|
||||||
|
public final int getFlags()
|
||||||
|
{
|
||||||
|
return flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public long getCrc() {
|
||||||
|
return crc;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public int getChecksumAlg() {
|
||||||
|
return checksumAlg;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void processCheckSum(LogBuffer buffer) {
|
||||||
|
if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF &&
|
||||||
|
checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){
|
||||||
|
crc = buffer.getUint32(eventLen - LogEvent.BINLOG_CHECKSUM_LEN);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @author jianghang 2013-4-8 上午12:36:29
|
||||||
|
* @version 1.0.3
|
||||||
|
* @since mysql 5.6
|
||||||
|
*/
|
||||||
|
public class PreviousGtidsLogEvent extends LogEvent{
|
||||||
|
|
||||||
|
public PreviousGtidsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header);
|
||||||
|
// do nothing , just for mysql gtid search function
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,920 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.charset.Charset;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.CharsetConversion;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A Query_log_event is created for each query that modifies the database,
|
||||||
|
* unless the query is logged row-based.
|
||||||
|
*
|
||||||
|
* The Post-Header has five components:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Post-Header for Query_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>slave_proxy_id</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>An integer identifying the client thread that issued the query. The id is
|
||||||
|
* unique per server. (Note, however, that two threads on different servers may
|
||||||
|
* have the same slave_proxy_id.) This is used when a client thread creates a
|
||||||
|
* temporary table local to the client. The slave_proxy_id is used to
|
||||||
|
* distinguish temporary tables that belong to different clients.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>exec_time</td>
|
||||||
|
* <td>4 byte unsigned integer</td>
|
||||||
|
* <td>The time from when the query started to when it was logged in the binlog,
|
||||||
|
* in seconds.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>db_len</td>
|
||||||
|
* <td>1 byte integer</td>
|
||||||
|
* <td>The length of the name of the currently selected database.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>error_code</td>
|
||||||
|
* <td>2 byte unsigned integer</td>
|
||||||
|
* <td>Error code generated by the master. If the master fails, the slave will
|
||||||
|
* fail with the same error code, except for the error codes ER_DB_CREATE_EXISTS
|
||||||
|
* == 1007 and ER_DB_DROP_EXISTS == 1008.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>status_vars_len</td>
|
||||||
|
* <td>2 byte unsigned integer</td>
|
||||||
|
* <td>The length of the status_vars block of the Body, in bytes. See
|
||||||
|
* query_log_event_status_vars "below".</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* The Body has the following components:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Body for Query_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>query_log_event_status_vars status_vars</td>
|
||||||
|
* <td>status_vars_len bytes</td>
|
||||||
|
* <td>Zero or more status variables. Each status variable consists of one byte
|
||||||
|
* identifying the variable stored, followed by the value of the variable. The
|
||||||
|
* possible variables are listed separately in the table
|
||||||
|
* Table_query_log_event_status_vars "below". MySQL always writes events in the
|
||||||
|
* order defined below; however, it is capable of reading them in any order.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>db</td>
|
||||||
|
* <td>db_len+1</td>
|
||||||
|
* <td>The currently selected database, as a null-terminated string.
|
||||||
|
*
|
||||||
|
* (The trailing zero is redundant since the length is already known; it is
|
||||||
|
* db_len from Post-Header.)</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>query</td>
|
||||||
|
* <td>variable length string without trailing zero, extending to the end of the
|
||||||
|
* event (determined by the length field of the Common-Header)</td>
|
||||||
|
* <td>The SQL query.</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* The following table lists the status variables that may appear in the
|
||||||
|
* status_vars field. Table_query_log_event_status_vars
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Status variables for Query_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Status variable</th>
|
||||||
|
* <th>1 byte identifier</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>flags2</td>
|
||||||
|
* <td>Q_FLAGS2_CODE == 0</td>
|
||||||
|
* <td>4 byte bitfield</td>
|
||||||
|
* <td>The flags in thd->options, binary AND-ed with OPTIONS_WRITTEN_TO_BIN_LOG.
|
||||||
|
* The thd->options bitfield contains options for "SELECT". OPTIONS_WRITTEN
|
||||||
|
* identifies those options that need to be written to the binlog (not all do).
|
||||||
|
* Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL |
|
||||||
|
* OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
|
||||||
|
* OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex.
|
||||||
|
*
|
||||||
|
* These flags correspond to the SQL variables SQL_AUTO_IS_NULL,
|
||||||
|
* FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and AUTOCOMMIT, documented in the
|
||||||
|
* "SET Syntax" section of the MySQL Manual.
|
||||||
|
*
|
||||||
|
* This field is always written to the binlog in version >= 5.0, and never
|
||||||
|
* written in version < 5.0.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>sql_mode</td>
|
||||||
|
* <td>Q_SQL_MODE_CODE == 1</td>
|
||||||
|
* <td>8 byte bitfield</td>
|
||||||
|
* <td>The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
|
||||||
|
* and see mysql_priv.h for a list of the possible flags. Currently
|
||||||
|
* (2007-10-04), the following flags are available:
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* MODE_REAL_AS_FLOAT==0x1
|
||||||
|
* MODE_PIPES_AS_CONCAT==0x2
|
||||||
|
* MODE_ANSI_QUOTES==0x4
|
||||||
|
* MODE_IGNORE_SPACE==0x8
|
||||||
|
* MODE_NOT_USED==0x10
|
||||||
|
* MODE_ONLY_FULL_GROUP_BY==0x20
|
||||||
|
* MODE_NO_UNSIGNED_SUBTRACTION==0x40
|
||||||
|
* MODE_NO_DIR_IN_CREATE==0x80
|
||||||
|
* MODE_POSTGRESQL==0x100
|
||||||
|
* MODE_ORACLE==0x200
|
||||||
|
* MODE_MSSQL==0x400
|
||||||
|
* MODE_DB2==0x800
|
||||||
|
* MODE_MAXDB==0x1000
|
||||||
|
* MODE_NO_KEY_OPTIONS==0x2000
|
||||||
|
* MODE_NO_TABLE_OPTIONS==0x4000
|
||||||
|
* MODE_NO_FIELD_OPTIONS==0x8000
|
||||||
|
* MODE_MYSQL323==0x10000
|
||||||
|
* MODE_MYSQL323==0x20000
|
||||||
|
* MODE_MYSQL40==0x40000
|
||||||
|
* MODE_ANSI==0x80000
|
||||||
|
* MODE_NO_AUTO_VALUE_ON_ZERO==0x100000
|
||||||
|
* MODE_NO_BACKSLASH_ESCAPES==0x200000
|
||||||
|
* MODE_STRICT_TRANS_TABLES==0x400000
|
||||||
|
* MODE_STRICT_ALL_TABLES==0x800000
|
||||||
|
* MODE_NO_ZERO_IN_DATE==0x1000000
|
||||||
|
* MODE_NO_ZERO_DATE==0x2000000
|
||||||
|
* MODE_INVALID_DATES==0x4000000
|
||||||
|
* MODE_ERROR_FOR_DIVISION_BY_ZERO==0x8000000
|
||||||
|
* MODE_TRADITIONAL==0x10000000
|
||||||
|
* MODE_NO_AUTO_CREATE_USER==0x20000000
|
||||||
|
* MODE_HIGH_NOT_PRECEDENCE==0x40000000
|
||||||
|
* MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* All these flags are replicated from the server. However, all flags except
|
||||||
|
* MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always preserves
|
||||||
|
* its old value of MODE_NO_DIR_IN_CREATE. For a rationale, see comment in
|
||||||
|
* Query_log_event::do_apply_event in log_event.cc.
|
||||||
|
*
|
||||||
|
* This field is always written to the binlog.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>catalog</td>
|
||||||
|
* <td>Q_CATALOG_NZ_CODE == 6</td>
|
||||||
|
* <td>Variable-length string: the length in bytes (1 byte) followed by the
|
||||||
|
* characters (at most 255 bytes)</td>
|
||||||
|
* <td>Stores the client's current catalog. Every database belongs to a catalog,
|
||||||
|
* the same way that every table belongs to a database. Currently, there is only
|
||||||
|
* one catalog, "std".
|
||||||
|
*
|
||||||
|
* This field is written if the length of the catalog is > 0; otherwise it is
|
||||||
|
* not written.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>auto_increment</td>
|
||||||
|
* <td>Q_AUTO_INCREMENT == 3</td>
|
||||||
|
* <td>two 2 byte unsigned integers, totally 2+2=4 bytes</td>
|
||||||
|
*
|
||||||
|
* <td>The two variables auto_increment_increment and auto_increment_offset, in
|
||||||
|
* that order. For more information, see "System variables" in the MySQL manual.
|
||||||
|
*
|
||||||
|
* This field is written if auto_increment > 1. Otherwise, it is not written.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>charset</td>
|
||||||
|
* <td>Q_CHARSET_CODE == 4</td>
|
||||||
|
* <td>three 2 byte unsigned integers, totally 2+2+2=6 bytes</td>
|
||||||
|
* <td>The three variables character_set_client, collation_connection, and
|
||||||
|
* collation_server, in that order. character_set_client is a code identifying
|
||||||
|
* the character set and collation used by the client to encode the query.
|
||||||
|
* collation_connection identifies the character set and collation that the
|
||||||
|
* master converts the query to when it receives it; this is useful when
|
||||||
|
* comparing literal strings. collation_server is the default character set and
|
||||||
|
* collation used when a new database is created.
|
||||||
|
*
|
||||||
|
* See also "Connection Character Sets and Collations" in the MySQL 5.1 manual.
|
||||||
|
*
|
||||||
|
* All three variables are codes identifying a (character set, collation) pair.
|
||||||
|
* To see which codes map to which pairs, run the query "SELECT id,
|
||||||
|
* character_set_name, collation_name FROM COLLATIONS".
|
||||||
|
*
|
||||||
|
* Cf. Q_CHARSET_DATABASE_CODE below.
|
||||||
|
*
|
||||||
|
* This field is always written.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>time_zone</td>
|
||||||
|
* <td>Q_TIME_ZONE_CODE == 5</td>
|
||||||
|
* <td>Variable-length string: the length in bytes (1 byte) followed by the
|
||||||
|
* characters (at most 255 bytes).
|
||||||
|
* <td>The time_zone of the master.
|
||||||
|
*
|
||||||
|
* See also "System Variables" and "MySQL Server Time Zone Support" in the MySQL
|
||||||
|
* manual.
|
||||||
|
*
|
||||||
|
* This field is written if the length of the time zone string is > 0;
|
||||||
|
* otherwise, it is not written.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>lc_time_names_number</td>
|
||||||
|
* <td>Q_LC_TIME_NAMES_CODE == 7</td>
|
||||||
|
* <td>2 byte integer</td>
|
||||||
|
* <td>A code identifying a table of month and day names. The mapping from codes
|
||||||
|
* to languages is defined in sql_locale.cc. This field is written if it is not
|
||||||
|
* 0, i.e., if the locale is not en_US.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>charset_database_number</td>
|
||||||
|
* <td>Q_CHARSET_DATABASE_CODE == 8</td>
|
||||||
|
* <td>2 byte integer</td>
|
||||||
|
*
|
||||||
|
* <td>The value of the collation_database system variable (in the source code
|
||||||
|
* stored in thd->variables.collation_database), which holds the code for a
|
||||||
|
* (character set, collation) pair as described above (see Q_CHARSET_CODE).
|
||||||
|
*
|
||||||
|
* collation_database was used in old versions (???WHEN). Its value was loaded
|
||||||
|
* when issuing a "use db" query and could be changed by issuing a
|
||||||
|
* "SET collation_database=xxx" query. It used to affect the "LOAD DATA INFILE"
|
||||||
|
* and "CREATE TABLE" commands.
|
||||||
|
*
|
||||||
|
* In newer versions, "CREATE TABLE" has been changed to take the character set
|
||||||
|
* from the database of the created table, rather than the character set of the
|
||||||
|
* current database. This makes a difference when creating a table in another
|
||||||
|
* database than the current one. "LOAD DATA INFILE" has not yet changed to do
|
||||||
|
* this, but there are plans to eventually do it, and to make collation_database
|
||||||
|
* read-only.
|
||||||
|
*
|
||||||
|
* This field is written if it is not 0.</td>
|
||||||
|
* </tr>
|
||||||
|
* <tr>
|
||||||
|
* <td>table_map_for_update</td>
|
||||||
|
* <td>Q_TABLE_MAP_FOR_UPDATE_CODE == 9</td>
|
||||||
|
* <td>8 byte integer</td>
|
||||||
|
*
|
||||||
|
* <td>The value of the table map that is to be updated by the multi-table
|
||||||
|
* update query statement. Every bit of this variable represents a table, and is
|
||||||
|
* set to 1 if the corresponding table is to be updated by this statement.
|
||||||
|
*
|
||||||
|
* The value of this variable is set when executing a multi-table update
|
||||||
|
* statement and used by slave to apply filter rules without opening all the
|
||||||
|
* tables on slave. This is required because some tables may not exist on slave
|
||||||
|
* because of the filter rules.</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* Query_log_event_notes_on_previous_versions Notes on Previous Versions
|
||||||
|
*
|
||||||
|
* Status vars were introduced in version 5.0. To read earlier versions
|
||||||
|
* correctly, check the length of the Post-Header.
|
||||||
|
*
|
||||||
|
* The status variable Q_CATALOG_CODE == 2 existed in MySQL 5.0.x, where
|
||||||
|
* 0<=x<=3. It was identical to Q_CATALOG_CODE, except that the string had a
|
||||||
|
* trailing '\0'. The '\0' was removed in 5.0.4 since it was redundant (the
|
||||||
|
* string length is stored before the string). The Q_CATALOG_CODE will never be
|
||||||
|
* written by a new master, but can still be understood by a new slave.
|
||||||
|
*
|
||||||
|
* See Q_CHARSET_DATABASE_CODE in the table above.
|
||||||
|
*
|
||||||
|
* When adding new status vars, please don't forget to update the
|
||||||
|
* MAX_SIZE_LOG_EVENT_STATUS, and update function code_name
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.6/sql/logevent.cc - Query_log_event
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public class QueryLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
The maximum number of updated databases that a status of
|
||||||
|
Query-log-event can carry. It can redefined within a range
|
||||||
|
[1.. OVER_MAX_DBS_IN_EVENT_MTS].
|
||||||
|
*/
|
||||||
|
public static final int MAX_DBS_IN_EVENT_MTS = 16;
|
||||||
|
|
||||||
|
/**
|
||||||
|
When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS
|
||||||
|
the value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the
|
||||||
|
mts_accessed_dbs status.
|
||||||
|
*/
|
||||||
|
public static final int OVER_MAX_DBS_IN_EVENT_MTS = 254;
|
||||||
|
|
||||||
|
|
||||||
|
public static final int SYSTEM_CHARSET_MBMAXLEN = 3;
|
||||||
|
public static final int NAME_CHAR_LEN = 64;
|
||||||
|
/* Field/table name length */
|
||||||
|
public static final int NAME_LEN = (NAME_CHAR_LEN * SYSTEM_CHARSET_MBMAXLEN);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Max number of possible extra bytes in a replication event compared to a
|
||||||
|
* packet (i.e. a query) sent from client to master; First, an auxiliary
|
||||||
|
* log_event status vars estimation:
|
||||||
|
*/
|
||||||
|
public static final int MAX_SIZE_LOG_EVENT_STATUS = (1 + 4 /* type, flags2 */
|
||||||
|
+ 1 + 8 /* type, sql_mode */
|
||||||
|
+ 1 + 1 + 255 /* type, length, catalog */
|
||||||
|
+ 1 + 4 /* type, auto_increment */
|
||||||
|
+ 1 + 6 /* type, charset */
|
||||||
|
+ 1 + 1 + 255 /* type, length, time_zone */
|
||||||
|
+ 1 + 2 /* type, lc_time_names_number */
|
||||||
|
+ 1 + 2 /* type, charset_database_number */
|
||||||
|
+ 1 + 8 /* type, table_map_for_update */
|
||||||
|
+ 1 + 4 /* type, master_data_written */
|
||||||
|
/* type, db_1, db_2, ... */
|
||||||
|
/* type, microseconds */
|
||||||
|
/* MariaDb type, sec_part of NOW() */
|
||||||
|
+ 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3
|
||||||
|
+ 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */);
|
||||||
|
/**
|
||||||
|
* Fixed data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>4 bytes. The ID of the thread that issued this statement. Needed for
|
||||||
|
* temporary tables. This is also useful for a DBA for knowing who did what
|
||||||
|
* on the master.</li>
|
||||||
|
* <li>4 bytes. The time in seconds that the statement took to execute. Only
|
||||||
|
* useful for inspection by the DBA.</li>
|
||||||
|
* <li>1 byte. The length of the name of the database which was the default
|
||||||
|
* database when the statement was executed. This name appears later, in the
|
||||||
|
* variable data part. It is necessary for statements such as INSERT INTO t
|
||||||
|
* VALUES(1) that don't specify the database and rely on the default
|
||||||
|
* database previously selected by USE.</li>
|
||||||
|
* <li>2 bytes. The error code resulting from execution of the statement on
|
||||||
|
* the master. Error codes are defined in include/mysqld_error.h. 0 means no
|
||||||
|
* error. How come statements with a non-zero error code can exist in the
|
||||||
|
* binary log? This is mainly due to the use of non-transactional tables
|
||||||
|
* within transactions. For example, if an INSERT ... SELECT fails after
|
||||||
|
* inserting 1000 rows into a MyISAM table (for example, with a
|
||||||
|
* duplicate-key violation), we have to write this statement to the binary
|
||||||
|
* log, because it truly modified the MyISAM table. For transactional
|
||||||
|
* tables, there should be no event with a non-zero error code (though it
|
||||||
|
* can happen, for example if the connection was interrupted (Control-C)).
|
||||||
|
* The slave checks the error code: After executing the statement itself, it
|
||||||
|
* compares the error code it got with the error code in the event, and if
|
||||||
|
* they are different it stops replicating (unless --slave-skip-errors was
|
||||||
|
* used to ignore the error).</li>
|
||||||
|
* <li>2 bytes (not present in v1, v3). The length of the status variable
|
||||||
|
* block.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Variable part:
|
||||||
|
* <ul>
|
||||||
|
* <li>Zero or more status variables (not present in v1, v3). Each status
|
||||||
|
* variable consists of one byte code identifying the variable stored,
|
||||||
|
* followed by the value of the variable. The format of the value is
|
||||||
|
* variable-specific, as described later.</li>
|
||||||
|
* <li>The default database name (null-terminated).</li>
|
||||||
|
* <li>The SQL statement. The slave knows the size of the other fields in
|
||||||
|
* the variable part (the sizes are given in the fixed data part), so by
|
||||||
|
* subtraction it can know the size of the statement.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
private String user;
|
||||||
|
private String host;
|
||||||
|
|
||||||
|
/* using byte for query string */
|
||||||
|
protected String query;
|
||||||
|
protected String catalog;
|
||||||
|
protected final String dbname;
|
||||||
|
|
||||||
|
/** The number of seconds the query took to run on the master. */
|
||||||
|
// The time in seconds that the statement took to execute. Only useful for inspection by the DBA
|
||||||
|
private final long execTime;
|
||||||
|
private final int errorCode;
|
||||||
|
private final long sessionId; /* thread_id */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 'flags2' is a second set of flags (on top of those in Log_event), for
|
||||||
|
* session variables. These are thd->options which is & against a mask
|
||||||
|
* (OPTIONS_WRITTEN_TO_BIN_LOG).
|
||||||
|
*/
|
||||||
|
private long flags2;
|
||||||
|
|
||||||
|
/** In connections sql_mode is 32 bits now but will be 64 bits soon */
|
||||||
|
private long sql_mode;
|
||||||
|
|
||||||
|
private long autoIncrementIncrement = -1;
|
||||||
|
private long autoIncrementOffset = -1;
|
||||||
|
|
||||||
|
private int clientCharset = -1;
|
||||||
|
private int clientCollation = -1;
|
||||||
|
private int serverCollation = -1;
|
||||||
|
private String charsetName;
|
||||||
|
|
||||||
|
private String timezone;
|
||||||
|
|
||||||
|
public QueryLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent) throws IOException
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
|
||||||
|
/*
|
||||||
|
* We test if the event's length is sensible, and if so we compute
|
||||||
|
* data_len. We cannot rely on QUERY_HEADER_LEN here as it would not be
|
||||||
|
* format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same
|
||||||
|
* for 3.23, 4.0 & 5.0.
|
||||||
|
*/
|
||||||
|
if (buffer.limit() < (commonHeaderLen + postHeaderLen))
|
||||||
|
{
|
||||||
|
throw new IOException("Query event length is too short.");
|
||||||
|
}
|
||||||
|
int dataLen = buffer.limit() - (commonHeaderLen + postHeaderLen);
|
||||||
|
buffer.position(commonHeaderLen + Q_THREAD_ID_OFFSET);
|
||||||
|
|
||||||
|
sessionId = buffer.getUint32(); // Q_THREAD_ID_OFFSET
|
||||||
|
execTime = buffer.getUint32(); // Q_EXEC_TIME_OFFSET
|
||||||
|
|
||||||
|
// TODO: add a check of all *_len vars
|
||||||
|
final int dbLen = buffer.getUint8(); // Q_DB_LEN_OFFSET
|
||||||
|
errorCode = buffer.getUint16(); // Q_ERR_CODE_OFFSET
|
||||||
|
|
||||||
|
/*
|
||||||
|
* 5.0 format starts here. Depending on the format, we may or not
|
||||||
|
* have affected/warnings etc The remaining post-header to be parsed
|
||||||
|
* has length:
|
||||||
|
*/
|
||||||
|
int statusVarsLen = 0;
|
||||||
|
if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN)
|
||||||
|
{
|
||||||
|
statusVarsLen = buffer.getUint16(); // Q_STATUS_VARS_LEN_OFFSET
|
||||||
|
/*
|
||||||
|
Check if status variable length is corrupt and will lead to very
|
||||||
|
wrong data. We could be even more strict and require data_len to
|
||||||
|
be even bigger, but this will suffice to catch most corruption
|
||||||
|
errors that can lead to a crash.
|
||||||
|
*/
|
||||||
|
if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS))
|
||||||
|
{
|
||||||
|
throw new IOException("status_vars_len (" + statusVarsLen
|
||||||
|
+ ") > data_len (" + dataLen + ")");
|
||||||
|
}
|
||||||
|
dataLen -= statusVarsLen;
|
||||||
|
}
|
||||||
|
/*
|
||||||
|
* We have parsed everything we know in the post header for QUERY_EVENT,
|
||||||
|
* the rest of post header is either comes from older version MySQL or
|
||||||
|
* dedicated to derived events (e.g. Execute_load_query...)
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* variable-part: the status vars; only in MySQL 5.0 */
|
||||||
|
final int start = commonHeaderLen + postHeaderLen;
|
||||||
|
final int limit = buffer.limit(); /* for restore */
|
||||||
|
final int end = start + statusVarsLen;
|
||||||
|
buffer.position(start).limit(end);
|
||||||
|
unpackVariables(buffer, end);
|
||||||
|
buffer.position(end);
|
||||||
|
buffer.limit(limit);
|
||||||
|
|
||||||
|
/* A 2nd variable part; this is common to all versions */
|
||||||
|
final int queryLen = dataLen - dbLen - 1;
|
||||||
|
dbname = buffer.getFixString(dbLen + 1);
|
||||||
|
if (clientCharset >= 0)
|
||||||
|
{
|
||||||
|
charsetName = CharsetConversion.getJavaCharset(clientCharset);
|
||||||
|
|
||||||
|
if ((charsetName != null) && (Charset.isSupported(charsetName)))
|
||||||
|
{
|
||||||
|
query = buffer.getFixString(queryLen, charsetName);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.warn("unsupported character set in query log: "
|
||||||
|
+ "\n ID = " + clientCharset + ", Charset = "
|
||||||
|
+ CharsetConversion.getCharset(clientCharset)
|
||||||
|
+ ", Collation = "
|
||||||
|
+ CharsetConversion.getCollation(clientCharset));
|
||||||
|
|
||||||
|
query = buffer.getFixString(queryLen);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
query = buffer.getFixString(queryLen);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* query event post-header */
|
||||||
|
public static final int Q_THREAD_ID_OFFSET = 0;
|
||||||
|
public static final int Q_EXEC_TIME_OFFSET = 4;
|
||||||
|
public static final int Q_DB_LEN_OFFSET = 8;
|
||||||
|
public static final int Q_ERR_CODE_OFFSET = 9;
|
||||||
|
public static final int Q_STATUS_VARS_LEN_OFFSET = 11;
|
||||||
|
public static final int Q_DATA_OFFSET = QUERY_HEADER_LEN;
|
||||||
|
|
||||||
|
/* these are codes, not offsets; not more than 256 values (1 byte). */
|
||||||
|
public static final int Q_FLAGS2_CODE = 0;
|
||||||
|
public static final int Q_SQL_MODE_CODE = 1;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Q_CATALOG_CODE is catalog with end zero stored; it is used only by MySQL
|
||||||
|
* 5.0.x where 0<=x<=3. We have to keep it to be able to replicate these old
|
||||||
|
* masters.
|
||||||
|
*/
|
||||||
|
public static final int Q_CATALOG_CODE = 2;
|
||||||
|
public static final int Q_AUTO_INCREMENT = 3;
|
||||||
|
public static final int Q_CHARSET_CODE = 4;
|
||||||
|
public static final int Q_TIME_ZONE_CODE = 5;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Q_CATALOG_NZ_CODE is catalog withOUT end zero stored; it is used by MySQL
|
||||||
|
* 5.0.x where x>=4. Saves one byte in every Query_log_event in binlog,
|
||||||
|
* compared to Q_CATALOG_CODE. The reason we didn't simply re-use
|
||||||
|
* Q_CATALOG_CODE is that then a 5.0.3 slave of this 5.0.x (x>=4) master
|
||||||
|
* would crash (segfault etc) because it would expect a 0 when there is
|
||||||
|
* none.
|
||||||
|
*/
|
||||||
|
public static final int Q_CATALOG_NZ_CODE = 6;
|
||||||
|
|
||||||
|
public static final int Q_LC_TIME_NAMES_CODE = 7;
|
||||||
|
|
||||||
|
public static final int Q_CHARSET_DATABASE_CODE = 8;
|
||||||
|
|
||||||
|
public static final int Q_TABLE_MAP_FOR_UPDATE_CODE = 9;
|
||||||
|
|
||||||
|
public static final int Q_MASTER_DATA_WRITTEN_CODE = 10;
|
||||||
|
|
||||||
|
public static final int Q_INVOKER = 11;
|
||||||
|
|
||||||
|
/**
|
||||||
|
Q_UPDATED_DB_NAMES status variable collects of the updated databases
|
||||||
|
total number and their names to be propagated to the slave in order
|
||||||
|
to facilitate the parallel applying of the Query events.
|
||||||
|
*/
|
||||||
|
public static final int Q_UPDATED_DB_NAMES = 12;
|
||||||
|
|
||||||
|
public static final int Q_MICROSECONDS = 13;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* FROM MariaDB 5.5.34
|
||||||
|
*/
|
||||||
|
public static final int Q_HRNOW = 128;
|
||||||
|
|
||||||
|
private final void unpackVariables(LogBuffer buffer, final int end)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
int code = -1;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
while (buffer.position() < end)
|
||||||
|
{
|
||||||
|
switch (code = buffer.getUint8())
|
||||||
|
{
|
||||||
|
case Q_FLAGS2_CODE:
|
||||||
|
flags2 = buffer.getUint32();
|
||||||
|
break;
|
||||||
|
case Q_SQL_MODE_CODE:
|
||||||
|
sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode is ulonglong
|
||||||
|
break;
|
||||||
|
case Q_CATALOG_NZ_CODE:
|
||||||
|
catalog = buffer.getString();
|
||||||
|
break;
|
||||||
|
case Q_AUTO_INCREMENT:
|
||||||
|
autoIncrementIncrement = buffer.getUint16();
|
||||||
|
autoIncrementOffset = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
case Q_CHARSET_CODE:
|
||||||
|
// Charset: 6 byte character set flag.
|
||||||
|
// 1-2 = character set client
|
||||||
|
// 3-4 = collation client
|
||||||
|
// 5-6 = collation server
|
||||||
|
clientCharset = buffer.getUint16();
|
||||||
|
clientCollation = buffer.getUint16();
|
||||||
|
serverCollation = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
case Q_TIME_ZONE_CODE:
|
||||||
|
timezone = buffer.getString();
|
||||||
|
break;
|
||||||
|
case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
|
||||||
|
final int len = buffer.getUint8();
|
||||||
|
catalog = buffer.getFixString(len + 1);
|
||||||
|
break;
|
||||||
|
case Q_LC_TIME_NAMES_CODE:
|
||||||
|
// lc_time_names_number = buffer.getUint16();
|
||||||
|
buffer.forward(2);
|
||||||
|
break;
|
||||||
|
case Q_CHARSET_DATABASE_CODE:
|
||||||
|
// charset_database_number = buffer.getUint16();
|
||||||
|
buffer.forward(2);
|
||||||
|
break;
|
||||||
|
case Q_TABLE_MAP_FOR_UPDATE_CODE:
|
||||||
|
// table_map_for_update = buffer.getUlong64();
|
||||||
|
buffer.forward(8);
|
||||||
|
break;
|
||||||
|
case Q_MASTER_DATA_WRITTEN_CODE:
|
||||||
|
// data_written = master_data_written = buffer.getUint32();
|
||||||
|
buffer.forward(4);
|
||||||
|
break;
|
||||||
|
case Q_INVOKER:
|
||||||
|
user = buffer.getString();
|
||||||
|
host = buffer.getString();
|
||||||
|
break;
|
||||||
|
case Q_MICROSECONDS:
|
||||||
|
// when.tv_usec= uint3korr(pos);
|
||||||
|
buffer.forward(3);
|
||||||
|
break;
|
||||||
|
case Q_UPDATED_DB_NAMES:
|
||||||
|
int mtsAccessedDbs = buffer.getUint8();
|
||||||
|
/**
|
||||||
|
Notice, the following check is positive also in case of
|
||||||
|
the master's MAX_DBS_IN_EVENT_MTS > the slave's one and the event
|
||||||
|
contains e.g the master's MAX_DBS_IN_EVENT_MTS db:s.
|
||||||
|
*/
|
||||||
|
if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) {
|
||||||
|
mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
|
||||||
|
for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
|
||||||
|
int length = end - buffer.position();
|
||||||
|
mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case Q_HRNOW:
|
||||||
|
// int when_sec_part = buffer.getUint24();
|
||||||
|
buffer.forward(3);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
/* That's why you must write status vars in growing order of code */
|
||||||
|
if (logger.isDebugEnabled())
|
||||||
|
logger.debug("Query_log_event has unknown status vars (first has code: "
|
||||||
|
+ code + "), skipping the rest of them");
|
||||||
|
break; // Break loop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (RuntimeException e)
|
||||||
|
{
|
||||||
|
throw new IOException("Read " + findCodeName(code) + " error: "
|
||||||
|
+ e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final String findCodeName(final int code)
|
||||||
|
{
|
||||||
|
switch (code)
|
||||||
|
{
|
||||||
|
case Q_FLAGS2_CODE:
|
||||||
|
return "Q_FLAGS2_CODE";
|
||||||
|
case Q_SQL_MODE_CODE:
|
||||||
|
return "Q_SQL_MODE_CODE";
|
||||||
|
case Q_CATALOG_CODE:
|
||||||
|
return "Q_CATALOG_CODE";
|
||||||
|
case Q_AUTO_INCREMENT:
|
||||||
|
return "Q_AUTO_INCREMENT";
|
||||||
|
case Q_CHARSET_CODE:
|
||||||
|
return "Q_CHARSET_CODE";
|
||||||
|
case Q_TIME_ZONE_CODE:
|
||||||
|
return "Q_TIME_ZONE_CODE";
|
||||||
|
case Q_CATALOG_NZ_CODE:
|
||||||
|
return "Q_CATALOG_NZ_CODE";
|
||||||
|
case Q_LC_TIME_NAMES_CODE:
|
||||||
|
return "Q_LC_TIME_NAMES_CODE";
|
||||||
|
case Q_CHARSET_DATABASE_CODE:
|
||||||
|
return "Q_CHARSET_DATABASE_CODE";
|
||||||
|
case Q_TABLE_MAP_FOR_UPDATE_CODE:
|
||||||
|
return "Q_TABLE_MAP_FOR_UPDATE_CODE";
|
||||||
|
case Q_MASTER_DATA_WRITTEN_CODE:
|
||||||
|
return "Q_MASTER_DATA_WRITTEN_CODE";
|
||||||
|
case Q_UPDATED_DB_NAMES:
|
||||||
|
return "Q_UPDATED_DB_NAMES";
|
||||||
|
case Q_MICROSECONDS:
|
||||||
|
return "Q_MICROSECONDS";
|
||||||
|
}
|
||||||
|
return "CODE#" + code;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getUser()
|
||||||
|
{
|
||||||
|
return user;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getHost()
|
||||||
|
{
|
||||||
|
return host;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getQuery()
|
||||||
|
{
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getCatalog()
|
||||||
|
{
|
||||||
|
return catalog;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getDbName()
|
||||||
|
{
|
||||||
|
return dbname;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The number of seconds the query took to run on the master.
|
||||||
|
*/
|
||||||
|
public final long getExecTime()
|
||||||
|
{
|
||||||
|
return execTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getErrorCode()
|
||||||
|
{
|
||||||
|
return errorCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getSessionId()
|
||||||
|
{
|
||||||
|
return sessionId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getAutoIncrementIncrement()
|
||||||
|
{
|
||||||
|
return autoIncrementIncrement;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getAutoIncrementOffset()
|
||||||
|
{
|
||||||
|
return autoIncrementOffset;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getCharsetName()
|
||||||
|
{
|
||||||
|
return charsetName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getTimezone()
|
||||||
|
{
|
||||||
|
return timezone;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the charsetID value.
|
||||||
|
*
|
||||||
|
* @return Returns the charsetID.
|
||||||
|
*/
|
||||||
|
public final int getClientCharset()
|
||||||
|
{
|
||||||
|
return clientCharset;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the clientCollationId value.
|
||||||
|
*
|
||||||
|
* @return Returns the clientCollationId.
|
||||||
|
*/
|
||||||
|
public final int getClientCollation()
|
||||||
|
{
|
||||||
|
return clientCollation;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the serverCollationId value.
|
||||||
|
*
|
||||||
|
* @return Returns the serverCollationId.
|
||||||
|
*/
|
||||||
|
public final int getServerCollation()
|
||||||
|
{
|
||||||
|
return serverCollation;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the sql_mode value.
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
|
||||||
|
* and see mysql_priv.h for a list of the possible flags. Currently
|
||||||
|
* (2007-10-04), the following flags are available:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>MODE_REAL_AS_FLOAT==0x1</li>
|
||||||
|
* <li>MODE_PIPES_AS_CONCAT==0x2</li>
|
||||||
|
* <li>MODE_ANSI_QUOTES==0x4</li>
|
||||||
|
* <li>MODE_IGNORE_SPACE==0x8</li>
|
||||||
|
* <li>MODE_NOT_USED==0x10</li>
|
||||||
|
* <li>MODE_ONLY_FULL_GROUP_BY==0x20</li>
|
||||||
|
* <li>MODE_NO_UNSIGNED_SUBTRACTION==0x40</li>
|
||||||
|
* <li>MODE_NO_DIR_IN_CREATE==0x80</li>
|
||||||
|
* <li>MODE_POSTGRESQL==0x100</li>
|
||||||
|
* <li>MODE_ORACLE==0x200</li>
|
||||||
|
* <li>MODE_MSSQL==0x400</li>
|
||||||
|
* <li>MODE_DB2==0x800</li>
|
||||||
|
* <li>MODE_MAXDB==0x1000</li>
|
||||||
|
* <li>MODE_NO_KEY_OPTIONS==0x2000</li>
|
||||||
|
* <li>MODE_NO_TABLE_OPTIONS==0x4000</li>
|
||||||
|
* <li>MODE_NO_FIELD_OPTIONS==0x8000</li>
|
||||||
|
* <li>MODE_MYSQL323==0x10000</li>
|
||||||
|
* <li>MODE_MYSQL40==0x20000</li>
|
||||||
|
* <li>MODE_ANSI==0x40000</li>
|
||||||
|
* <li>MODE_NO_AUTO_VALUE_ON_ZERO==0x80000</li>
|
||||||
|
* <li>MODE_NO_BACKSLASH_ESCAPES==0x100000</li>
|
||||||
|
* <li>MODE_STRICT_TRANS_TABLES==0x200000</li>
|
||||||
|
* <li>MODE_STRICT_ALL_TABLES==0x400000</li>
|
||||||
|
* <li>MODE_NO_ZERO_IN_DATE==0x800000</li>
|
||||||
|
* <li>MODE_NO_ZERO_DATE==0x1000000</li>
|
||||||
|
* <li>MODE_INVALID_DATES==0x2000000</li>
|
||||||
|
* <li>MODE_ERROR_FOR_DIVISION_BY_ZERO==0x4000000</li>
|
||||||
|
* <li>MODE_TRADITIONAL==0x8000000</li>
|
||||||
|
* <li>MODE_NO_AUTO_CREATE_USER==0x10000000</li>
|
||||||
|
* <li>MODE_HIGH_NOT_PRECEDENCE==0x20000000</li>
|
||||||
|
* <li>MODE_NO_ENGINE_SUBSTITUTION=0x40000000</li>
|
||||||
|
* <li>MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* All these flags are replicated from the server. However, all flags except
|
||||||
|
* MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always
|
||||||
|
* preserves its old value of MODE_NO_DIR_IN_CREATE. This field is always
|
||||||
|
* written to the binlog.
|
||||||
|
*/
|
||||||
|
public final long getSqlMode()
|
||||||
|
{
|
||||||
|
return sql_mode;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* FLAGS2 values that can be represented inside the binlog */
|
||||||
|
public static final int OPTION_AUTO_IS_NULL = 1 << 14;
|
||||||
|
public static final int OPTION_NOT_AUTOCOMMIT = 1 << 19;
|
||||||
|
public static final int OPTION_NO_FOREIGN_KEY_CHECKS = 1 << 26;
|
||||||
|
public static final int OPTION_RELAXED_UNIQUE_CHECKS = 1 << 27;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The flags in thd->options, binary AND-ed with OPTIONS_WRITTEN_TO_BIN_LOG.
|
||||||
|
* The thd->options bitfield contains options for "SELECT". OPTIONS_WRITTEN
|
||||||
|
* identifies those options that need to be written to the binlog (not all
|
||||||
|
* do). Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL
|
||||||
|
* | OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
|
||||||
|
* OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. These flags correspond to
|
||||||
|
* the SQL variables SQL_AUTO_IS_NULL, FOREIGN_KEY_CHECKS, UNIQUE_CHECKS,
|
||||||
|
* and AUTOCOMMIT, documented in the "SET Syntax" section of the MySQL
|
||||||
|
* Manual. This field is always written to the binlog in version >= 5.0, and
|
||||||
|
* never written in version < 5.0.
|
||||||
|
*/
|
||||||
|
public final long getFlags2()
|
||||||
|
{
|
||||||
|
return flags2;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the OPTION_AUTO_IS_NULL flag.
|
||||||
|
*/
|
||||||
|
public final boolean isAutoIsNull()
|
||||||
|
{
|
||||||
|
return ((flags2 & OPTION_AUTO_IS_NULL) == OPTION_AUTO_IS_NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
|
||||||
|
*/
|
||||||
|
public final boolean isForeignKeyChecks()
|
||||||
|
{
|
||||||
|
return ((flags2 & OPTION_NO_FOREIGN_KEY_CHECKS) != OPTION_NO_FOREIGN_KEY_CHECKS);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the OPTION_NOT_AUTOCOMMIT flag.
|
||||||
|
*/
|
||||||
|
public final boolean isAutocommit()
|
||||||
|
{
|
||||||
|
return ((flags2 & OPTION_NOT_AUTOCOMMIT) != OPTION_NOT_AUTOCOMMIT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
|
||||||
|
*/
|
||||||
|
public final boolean isUniqueChecks()
|
||||||
|
{
|
||||||
|
return ((flags2 & OPTION_RELAXED_UNIQUE_CHECKS) != OPTION_RELAXED_UNIQUE_CHECKS);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1
|
||||||
|
* does not need it (it's repeatable again) so this event needn't be written in
|
||||||
|
* 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it
|
||||||
|
* does not cause bugs).
|
||||||
|
*
|
||||||
|
* The state of the random number generation consists of 128 bits, which are
|
||||||
|
* stored internally as two 64-bit numbers.
|
||||||
|
*
|
||||||
|
* Binary Format
|
||||||
|
*
|
||||||
|
* The Post-Header for this event type is empty. The Body has two components:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Body for Rand_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>seed1</td>
|
||||||
|
* <td>8 byte unsigned integer</td>
|
||||||
|
* <td>64 bit random seed1.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>seed2</td>
|
||||||
|
* <td>8 byte unsigned integer</td>
|
||||||
|
* <td>64 bit random seed2.</td>
|
||||||
|
* </tr>
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class RandLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Fixed data part: Empty
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Variable data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>8 bytes. The value for the first seed.</li>
|
||||||
|
* <li>8 bytes. The value for the second seed.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
private final long seed1;
|
||||||
|
private final long seed2;
|
||||||
|
|
||||||
|
/* Rand event data */
|
||||||
|
public static final int RAND_SEED1_OFFSET = 0;
|
||||||
|
public static final int RAND_SEED2_OFFSET = 8;
|
||||||
|
|
||||||
|
public RandLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
/* The Post-Header is empty. The Variable Data part begins immediately. */
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen
|
||||||
|
+ descriptionEvent.postHeaderLen[RAND_EVENT - 1]
|
||||||
|
+ RAND_SEED1_OFFSET);
|
||||||
|
seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET);
|
||||||
|
seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getQuery()
|
||||||
|
{
|
||||||
|
return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This will be deprecated when we move to using sequence ids.
|
||||||
|
*
|
||||||
|
* Binary Format
|
||||||
|
*
|
||||||
|
* The Post-Header has one component:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Post-Header for Rotate_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>position</td>
|
||||||
|
* <td>8 byte integer</td>
|
||||||
|
* <td>The position within the binlog to rotate to.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* The Body has one component:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Body for Rotate_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>new_log</td>
|
||||||
|
* <td>variable length string without trailing zero, extending to the end of the
|
||||||
|
* event (determined by the length field of the Common-Header)</td>
|
||||||
|
* <td>Name of the binlog to rotate to.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class RotateLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Fixed data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>8 bytes. The position of the first event in the next log file. Always
|
||||||
|
* contains the number 4 (meaning the next event starts at position 4 in the
|
||||||
|
* next binary log). This field is not present in v1; presumably the value
|
||||||
|
* is assumed to be 4.</li>
|
||||||
|
* </ul>
|
||||||
|
* <p>
|
||||||
|
*
|
||||||
|
* Variable data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>The name of the next binary log. The filename is not null-terminated.
|
||||||
|
* Its length is the event size minus the size of the fixed parts.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
private final String filename;
|
||||||
|
private final long position;
|
||||||
|
|
||||||
|
/* Rotate event post-header */
|
||||||
|
public static final int R_POS_OFFSET = 0;
|
||||||
|
public static final int R_IDENT_OFFSET = 8;
|
||||||
|
|
||||||
|
/* Max length of full path-name */
|
||||||
|
public static final int FN_REFLEN = 512;
|
||||||
|
|
||||||
|
// Rotate header with all empty fields.
|
||||||
|
public static final LogHeader ROTATE_HEADER = new LogHeader(ROTATE_EVENT);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new <code>Rotate_log_event</code> object read normally from
|
||||||
|
* log.
|
||||||
|
*
|
||||||
|
* @throws MySQLExtractException
|
||||||
|
*/
|
||||||
|
public RotateLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int headerSize = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1];
|
||||||
|
|
||||||
|
buffer.position(headerSize + R_POS_OFFSET);
|
||||||
|
position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET)
|
||||||
|
|
||||||
|
final int filenameOffset = headerSize + postHeaderLen;
|
||||||
|
int filenameLen = buffer.limit() - filenameOffset;
|
||||||
|
if (filenameLen > FN_REFLEN - 1)
|
||||||
|
filenameLen = FN_REFLEN - 1;
|
||||||
|
buffer.position(filenameOffset);
|
||||||
|
filename = buffer.getFixString(filenameLen);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new <code>Rotate_log_event</code> without log information. This
|
||||||
|
* is used to generate missing log rotation events.
|
||||||
|
*/
|
||||||
|
public RotateLogEvent(String filename)
|
||||||
|
{
|
||||||
|
super(ROTATE_HEADER);
|
||||||
|
|
||||||
|
this.filename = filename;
|
||||||
|
this.position = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new <code>Rotate_log_event</code> without log information.
|
||||||
|
*/
|
||||||
|
public RotateLogEvent(String filename, final long position)
|
||||||
|
{
|
||||||
|
super(ROTATE_HEADER);
|
||||||
|
|
||||||
|
this.filename = filename;
|
||||||
|
this.position = position;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getFilename()
|
||||||
|
{
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getPosition()
|
||||||
|
{
|
||||||
|
return position;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,976 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.sql.Timestamp;
|
||||||
|
import java.sql.Types;
|
||||||
|
import java.util.BitSet;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracting JDBC type & value information from packed rows-buffer.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc - Rows_log_event::print_verbose_one_row
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class RowsLogBuffer {
|
||||||
|
|
||||||
|
protected static final Log logger = LogFactory.getLog(RowsLogBuffer.class);
|
||||||
|
|
||||||
|
public static final long DATETIMEF_INT_OFS = 0x8000000000L;
|
||||||
|
public static final long TIMEF_INT_OFS = 0x800000L;
|
||||||
|
private final LogBuffer buffer;
|
||||||
|
private final int columnLen;
|
||||||
|
private final String charsetName;
|
||||||
|
// private Calendar cal;
|
||||||
|
|
||||||
|
private final BitSet nullBits;
|
||||||
|
private int nullBitIndex;
|
||||||
|
|
||||||
|
private boolean fNull;
|
||||||
|
private int javaType;
|
||||||
|
private int length;
|
||||||
|
private Serializable value;
|
||||||
|
|
||||||
|
public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName){
|
||||||
|
this.buffer = buffer;
|
||||||
|
this.columnLen = columnLen;
|
||||||
|
this.charsetName = charsetName;
|
||||||
|
this.nullBits = new BitSet(columnLen);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracting next row from packed buffer.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc -
|
||||||
|
* Rows_log_event::print_verbose_one_row
|
||||||
|
*/
|
||||||
|
public final boolean nextOneRow(BitSet columns) {
|
||||||
|
final boolean hasOneRow = buffer.hasRemaining();
|
||||||
|
|
||||||
|
if (hasOneRow) {
|
||||||
|
int column = 0;
|
||||||
|
|
||||||
|
for (int i = 0; i < columnLen; i++)
|
||||||
|
if (columns.get(i)) column++;
|
||||||
|
|
||||||
|
nullBitIndex = 0;
|
||||||
|
nullBits.clear();
|
||||||
|
buffer.fillBitmap(nullBits, column);
|
||||||
|
}
|
||||||
|
return hasOneRow;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracting next field value from packed buffer.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc -
|
||||||
|
* Rows_log_event::print_verbose_one_row
|
||||||
|
*/
|
||||||
|
public final Serializable nextValue(final int type, final int meta) {
|
||||||
|
return nextValue(type, meta, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracting next field value from packed buffer.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc -
|
||||||
|
* Rows_log_event::print_verbose_one_row
|
||||||
|
*/
|
||||||
|
public final Serializable nextValue(final int type, final int meta, boolean isBinary) {
|
||||||
|
fNull = nullBits.get(nullBitIndex++);
|
||||||
|
|
||||||
|
if (fNull) {
|
||||||
|
value = null;
|
||||||
|
javaType = mysqlToJavaType(type, meta, isBinary);
|
||||||
|
length = 0;
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
// Extracting field value from packed buffer.
|
||||||
|
return fetchValue(type, meta, isBinary);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maps the given MySQL type to the correct JDBC type.
|
||||||
|
*/
|
||||||
|
static int mysqlToJavaType(int type, final int meta, boolean isBinary) {
|
||||||
|
int javaType;
|
||||||
|
|
||||||
|
if (type == LogEvent.MYSQL_TYPE_STRING) {
|
||||||
|
if (meta >= 256) {
|
||||||
|
int byte0 = meta >> 8;
|
||||||
|
if ((byte0 & 0x30) != 0x30) {
|
||||||
|
/* a long CHAR() field: see #37426 */
|
||||||
|
type = byte0 | 0x30;
|
||||||
|
} else {
|
||||||
|
switch (byte0) {
|
||||||
|
case LogEvent.MYSQL_TYPE_SET:
|
||||||
|
case LogEvent.MYSQL_TYPE_ENUM:
|
||||||
|
case LogEvent.MYSQL_TYPE_STRING:
|
||||||
|
type = byte0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case LogEvent.MYSQL_TYPE_LONG:
|
||||||
|
javaType = Types.INTEGER;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_TINY:
|
||||||
|
javaType = Types.TINYINT;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_SHORT:
|
||||||
|
javaType = Types.SMALLINT;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_INT24:
|
||||||
|
javaType = Types.INTEGER;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_LONGLONG:
|
||||||
|
javaType = Types.BIGINT;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_DECIMAL:
|
||||||
|
javaType = Types.DECIMAL;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_NEWDECIMAL:
|
||||||
|
javaType = Types.DECIMAL;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_FLOAT:
|
||||||
|
javaType = Types.REAL; // Types.FLOAT;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_DOUBLE:
|
||||||
|
javaType = Types.DOUBLE;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_BIT:
|
||||||
|
javaType = Types.BIT;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_TIMESTAMP:
|
||||||
|
case LogEvent.MYSQL_TYPE_DATETIME:
|
||||||
|
javaType = Types.TIMESTAMP;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_TIME:
|
||||||
|
javaType = Types.TIME;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_NEWDATE:
|
||||||
|
case LogEvent.MYSQL_TYPE_DATE:
|
||||||
|
javaType = Types.DATE;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_YEAR:
|
||||||
|
javaType = Types.VARCHAR;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_ENUM:
|
||||||
|
javaType = Types.INTEGER;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_SET:
|
||||||
|
javaType = Types.BINARY;
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_TINY_BLOB:
|
||||||
|
case LogEvent.MYSQL_TYPE_MEDIUM_BLOB:
|
||||||
|
case LogEvent.MYSQL_TYPE_LONG_BLOB:
|
||||||
|
case LogEvent.MYSQL_TYPE_BLOB:
|
||||||
|
if (meta == 1) {
|
||||||
|
javaType = Types.VARBINARY;
|
||||||
|
} else {
|
||||||
|
javaType = Types.LONGVARBINARY;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_VARCHAR:
|
||||||
|
case LogEvent.MYSQL_TYPE_VAR_STRING:
|
||||||
|
if (isBinary) {
|
||||||
|
// varbinary在binlog中为var_string类型
|
||||||
|
javaType = Types.VARBINARY;
|
||||||
|
} else {
|
||||||
|
javaType = Types.VARCHAR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_STRING:
|
||||||
|
if (isBinary) {
|
||||||
|
// binary在binlog中为string类型
|
||||||
|
javaType = Types.BINARY;
|
||||||
|
} else {
|
||||||
|
javaType = Types.CHAR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case LogEvent.MYSQL_TYPE_GEOMETRY:
|
||||||
|
javaType = Types.BINARY;
|
||||||
|
break;
|
||||||
|
|
||||||
|
// case LogEvent.MYSQL_TYPE_BINARY:
|
||||||
|
// javaType = Types.BINARY;
|
||||||
|
// break;
|
||||||
|
//
|
||||||
|
// case LogEvent.MYSQL_TYPE_VARBINARY:
|
||||||
|
// javaType = Types.VARBINARY;
|
||||||
|
// break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
javaType = Types.OTHER;
|
||||||
|
}
|
||||||
|
|
||||||
|
return javaType;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracting next field value from packed buffer.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc - log_event_print_value
|
||||||
|
*/
|
||||||
|
final Serializable fetchValue(int type, final int meta, boolean isBinary) {
|
||||||
|
int len = 0;
|
||||||
|
|
||||||
|
if (type == LogEvent.MYSQL_TYPE_STRING) {
|
||||||
|
if (meta >= 256) {
|
||||||
|
int byte0 = meta >> 8;
|
||||||
|
int byte1 = meta & 0xff;
|
||||||
|
if ((byte0 & 0x30) != 0x30) {
|
||||||
|
/* a long CHAR() field: see #37426 */
|
||||||
|
len = byte1 | (((byte0 & 0x30) ^ 0x30) << 4);
|
||||||
|
type = byte0 | 0x30;
|
||||||
|
} else {
|
||||||
|
switch (byte0) {
|
||||||
|
case LogEvent.MYSQL_TYPE_SET:
|
||||||
|
case LogEvent.MYSQL_TYPE_ENUM:
|
||||||
|
case LogEvent.MYSQL_TYPE_STRING:
|
||||||
|
type = byte0;
|
||||||
|
len = byte1;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)",
|
||||||
|
type,
|
||||||
|
meta,
|
||||||
|
meta));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
len = meta;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (type) {
|
||||||
|
case LogEvent.MYSQL_TYPE_LONG: {
|
||||||
|
// XXX: How to check signed / unsigned?
|
||||||
|
// value = unsigned ? Long.valueOf(buffer.getUint32()) :
|
||||||
|
// Integer.valueOf(buffer.getInt32());
|
||||||
|
value = Integer.valueOf(buffer.getInt32());
|
||||||
|
javaType = Types.INTEGER;
|
||||||
|
length = 4;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_TINY: {
|
||||||
|
// XXX: How to check signed / unsigned?
|
||||||
|
// value = Integer.valueOf(unsigned ? buffer.getUint8() :
|
||||||
|
// buffer.getInt8());
|
||||||
|
value = Integer.valueOf(buffer.getInt8());
|
||||||
|
javaType = Types.TINYINT; // java.sql.Types.INTEGER;
|
||||||
|
length = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_SHORT: {
|
||||||
|
// XXX: How to check signed / unsigned?
|
||||||
|
// value = Integer.valueOf(unsigned ? buffer.getUint16() :
|
||||||
|
// buffer.getInt16());
|
||||||
|
value = Integer.valueOf((short) buffer.getInt16());
|
||||||
|
javaType = Types.SMALLINT; // java.sql.Types.INTEGER;
|
||||||
|
length = 2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_INT24: {
|
||||||
|
// XXX: How to check signed / unsigned?
|
||||||
|
// value = Integer.valueOf(unsigned ? buffer.getUint24() :
|
||||||
|
// buffer.getInt24());
|
||||||
|
value = Integer.valueOf(buffer.getInt24());
|
||||||
|
javaType = Types.INTEGER;
|
||||||
|
length = 3;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_LONGLONG: {
|
||||||
|
// XXX: How to check signed / unsigned?
|
||||||
|
// value = unsigned ? buffer.getUlong64()) :
|
||||||
|
// Long.valueOf(buffer.getLong64());
|
||||||
|
value = Long.valueOf(buffer.getLong64());
|
||||||
|
javaType = Types.BIGINT; // Types.INTEGER;
|
||||||
|
length = 8;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_DECIMAL: {
|
||||||
|
/*
|
||||||
|
* log_event.h : This enumeration value is only used internally
|
||||||
|
* and cannot exist in a binlog.
|
||||||
|
*/
|
||||||
|
logger.warn("MYSQL_TYPE_DECIMAL : This enumeration value is "
|
||||||
|
+ "only used internally and cannot exist in a binlog!");
|
||||||
|
javaType = Types.DECIMAL;
|
||||||
|
value = null; /* unknown format */
|
||||||
|
length = 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_NEWDECIMAL: {
|
||||||
|
final int precision = meta >> 8;
|
||||||
|
final int decimals = meta & 0xff;
|
||||||
|
value = buffer.getDecimal(precision, decimals);
|
||||||
|
javaType = Types.DECIMAL;
|
||||||
|
length = precision;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_FLOAT: {
|
||||||
|
value = Float.valueOf(buffer.getFloat32());
|
||||||
|
javaType = Types.REAL; // Types.FLOAT;
|
||||||
|
length = 4;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_DOUBLE: {
|
||||||
|
value = Double.valueOf(buffer.getDouble64());
|
||||||
|
javaType = Types.DOUBLE;
|
||||||
|
length = 8;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_BIT: {
|
||||||
|
/* Meta-data: bit_len, bytes_in_rec, 2 bytes */
|
||||||
|
final int nbits = ((meta >> 8) * 8) + (meta & 0xff);
|
||||||
|
len = (nbits + 7) / 8;
|
||||||
|
if (nbits > 1) {
|
||||||
|
// byte[] bits = new byte[len];
|
||||||
|
// buffer.fillBytes(bits, 0, len);
|
||||||
|
// 转化为unsign long
|
||||||
|
switch (len) {
|
||||||
|
case 1:
|
||||||
|
value = buffer.getInt8();
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
value = buffer.getBeUint16();
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
value = buffer.getBeUint24();
|
||||||
|
break;
|
||||||
|
case 4:
|
||||||
|
value = buffer.getBeUint32();
|
||||||
|
break;
|
||||||
|
case 5:
|
||||||
|
value = buffer.getBeUlong40();
|
||||||
|
break;
|
||||||
|
case 6:
|
||||||
|
value = buffer.getBeUlong48();
|
||||||
|
break;
|
||||||
|
case 7:
|
||||||
|
value = buffer.getBeUlong56();
|
||||||
|
break;
|
||||||
|
case 8:
|
||||||
|
value = buffer.getBeUlong64();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("!! Unknown Bit len = " + len);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
final int bit = buffer.getInt8();
|
||||||
|
// value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE;
|
||||||
|
value = bit;
|
||||||
|
}
|
||||||
|
javaType = Types.BIT;
|
||||||
|
length = nbits;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_TIMESTAMP: {
|
||||||
|
// MYSQL DataTypes: TIMESTAMP
|
||||||
|
// range is '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07'
|
||||||
|
// UTC
|
||||||
|
// A TIMESTAMP cannot represent the value '1970-01-01 00:00:00'
|
||||||
|
// because that is equivalent to 0 seconds from the epoch and
|
||||||
|
// the value 0 is reserved for representing '0000-00-00
|
||||||
|
// 00:00:00', the “zero” TIMESTAMP value.
|
||||||
|
final long i32 = buffer.getUint32();
|
||||||
|
if (i32 == 0) {
|
||||||
|
value = "0000-00-00 00:00:00";
|
||||||
|
} else {
|
||||||
|
String v = new Timestamp(i32 * 1000).toString();
|
||||||
|
value = v.substring(0, v.length() - 2);
|
||||||
|
}
|
||||||
|
javaType = Types.TIMESTAMP;
|
||||||
|
length = 4;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_TIMESTAMP2: {
|
||||||
|
final long tv_sec = buffer.getBeUint32(); // big-endian
|
||||||
|
int tv_usec = 0;
|
||||||
|
switch (meta) {
|
||||||
|
case 0:
|
||||||
|
tv_usec = 0;
|
||||||
|
break;
|
||||||
|
case 1:
|
||||||
|
case 2:
|
||||||
|
tv_usec = buffer.getInt8() * 10000;
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
case 4:
|
||||||
|
tv_usec = buffer.getBeInt16() * 100;
|
||||||
|
break;
|
||||||
|
case 5:
|
||||||
|
case 6:
|
||||||
|
tv_usec = buffer.getBeInt24();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
tv_usec = 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tv_sec == 0) {
|
||||||
|
value = "0000-00-00 00:00:00";
|
||||||
|
} else {
|
||||||
|
Timestamp time = new Timestamp(tv_sec * 1000);
|
||||||
|
time.setNanos(tv_usec * 1000);
|
||||||
|
String v = time.toString();
|
||||||
|
value = v.substring(0, v.length() - 2);
|
||||||
|
}
|
||||||
|
javaType = Types.TIMESTAMP;
|
||||||
|
length = 4 + (meta + 1) / 2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_DATETIME: {
|
||||||
|
// MYSQL DataTypes: DATETIME
|
||||||
|
// range is '0000-01-01 00:00:00' to '9999-12-31 23:59:59'
|
||||||
|
final long i64 = buffer.getLong64(); /* YYYYMMDDhhmmss */
|
||||||
|
if (i64 == 0) {
|
||||||
|
value = "0000-00-00 00:00:00";
|
||||||
|
} else {
|
||||||
|
final int d = (int) (i64 / 1000000);
|
||||||
|
final int t = (int) (i64 % 1000000);
|
||||||
|
// if (cal == null) cal = Calendar.getInstance();
|
||||||
|
// cal.clear();
|
||||||
|
/* month is 0-based, 0 for january. */
|
||||||
|
// cal.set(d / 10000, (d % 10000) / 100 - 1, d % 100, t /
|
||||||
|
// 10000, (t % 10000) / 100, t % 100);
|
||||||
|
// value = new Timestamp(cal.getTimeInMillis());
|
||||||
|
value = String.format("%04d-%02d-%02d %02d:%02d:%02d",
|
||||||
|
d / 10000,
|
||||||
|
(d % 10000) / 100,
|
||||||
|
d % 100,
|
||||||
|
t / 10000,
|
||||||
|
(t % 10000) / 100,
|
||||||
|
t % 100);
|
||||||
|
}
|
||||||
|
javaType = Types.TIMESTAMP;
|
||||||
|
length = 8;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_DATETIME2: {
|
||||||
|
/*
|
||||||
|
* DATETIME and DATE low-level memory and disk representation
|
||||||
|
* routines 1 bit sign (used when on disk) 17 bits year*13+month
|
||||||
|
* (year 0-9999, month 0-12) 5 bits day (0-31) 5 bits hour
|
||||||
|
* (0-23) 6 bits minute (0-59) 6 bits second (0-59) 24 bits
|
||||||
|
* microseconds (0-999999) Total: 64 bits = 8 bytes
|
||||||
|
* SYYYYYYY.YYYYYYYY
|
||||||
|
* .YYdddddh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff
|
||||||
|
*/
|
||||||
|
long intpart = buffer.getBeUlong40() - DATETIMEF_INT_OFS; // big-endian
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
int frac = 0;
|
||||||
|
switch (meta) {
|
||||||
|
case 0:
|
||||||
|
frac = 0;
|
||||||
|
break;
|
||||||
|
case 1:
|
||||||
|
case 2:
|
||||||
|
frac = buffer.getInt8() * 10000;
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
case 4:
|
||||||
|
frac = buffer.getBeInt16() * 100;
|
||||||
|
break;
|
||||||
|
case 5:
|
||||||
|
case 6:
|
||||||
|
frac = buffer.getBeInt24();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
frac = 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (intpart == 0) {
|
||||||
|
value = "0000-00-00 00:00:00";
|
||||||
|
} else {
|
||||||
|
// 构造TimeStamp只处理到秒
|
||||||
|
long ymd = intpart >> 17;
|
||||||
|
long ym = ymd >> 5;
|
||||||
|
long hms = intpart % (1 << 17);
|
||||||
|
|
||||||
|
// if (cal == null) cal = Calendar.getInstance();
|
||||||
|
// cal.clear();
|
||||||
|
// cal.set((int) (ym / 13), (int) (ym % 13) - 1, (int) (ymd
|
||||||
|
// % (1 << 5)), (int) (hms >> 12),
|
||||||
|
// (int) ((hms >> 6) % (1 << 6)), (int) (hms % (1 << 6)));
|
||||||
|
// value = new Timestamp(cal.getTimeInMillis());
|
||||||
|
value = String.format("%04d-%02d-%02d %02d:%02d:%02d",
|
||||||
|
(int) (ym / 13),
|
||||||
|
(int) (ym % 13),
|
||||||
|
(int) (ymd % (1 << 5)),
|
||||||
|
(int) (hms >> 12),
|
||||||
|
(int) ((hms >> 6) % (1 << 6)),
|
||||||
|
(int) (hms % (1 << 6)));
|
||||||
|
}
|
||||||
|
javaType = Types.TIMESTAMP;
|
||||||
|
length = 5 + (meta + 1) / 2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_TIME: {
|
||||||
|
// MYSQL DataTypes: TIME
|
||||||
|
// The range is '-838:59:59' to '838:59:59'
|
||||||
|
// final int i32 = buffer.getUint24();
|
||||||
|
final int i32 = buffer.getInt24();
|
||||||
|
final int u32 = Math.abs(i32);
|
||||||
|
if (i32 == 0) {
|
||||||
|
value = "00:00:00";
|
||||||
|
} else {
|
||||||
|
// if (cal == null) cal = Calendar.getInstance();
|
||||||
|
// cal.clear();
|
||||||
|
// cal.set(70, 0, 1, i32 / 10000, (i32 % 10000) / 100, i32 %
|
||||||
|
// 100);
|
||||||
|
// value = new Time(cal.getTimeInMillis());
|
||||||
|
value = String.format("%s%02d:%02d:%02d",
|
||||||
|
(i32 >= 0) ? "" : "-",
|
||||||
|
u32 / 10000,
|
||||||
|
(u32 % 10000) / 100,
|
||||||
|
u32 % 100);
|
||||||
|
}
|
||||||
|
javaType = Types.TIME;
|
||||||
|
length = 3;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_TIME2: {
|
||||||
|
/*
|
||||||
|
* TIME low-level memory and disk representation routines
|
||||||
|
* In-memory format: 1 bit sign (Used for sign, when on disk) 1
|
||||||
|
* bit unused (Reserved for wider hour range, e.g. for
|
||||||
|
* intervals) 10 bit hour (0-836) 6 bit minute (0-59) 6 bit
|
||||||
|
* second (0-59) 24 bits microseconds (0-999999) Total: 48 bits
|
||||||
|
* = 6 bytes
|
||||||
|
* Suhhhhhh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff
|
||||||
|
*/
|
||||||
|
long intpart = 0;
|
||||||
|
int frac = 0;
|
||||||
|
long ltime = 0;
|
||||||
|
switch (meta) {
|
||||||
|
case 0:
|
||||||
|
intpart = buffer.getBeUint24() - TIMEF_INT_OFS; // big-endian
|
||||||
|
ltime = intpart << 24;
|
||||||
|
break;
|
||||||
|
case 1:
|
||||||
|
case 2:
|
||||||
|
intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
|
||||||
|
frac = buffer.getUint8();
|
||||||
|
if (intpart < 0 && frac > 0) {
|
||||||
|
/*
|
||||||
|
* Negative values are stored with reverse
|
||||||
|
* fractional part order, for binary sort
|
||||||
|
* compatibility. Disk value intpart frac Time value
|
||||||
|
* Memory value 800000.00 0 0 00:00:00.00
|
||||||
|
* 0000000000.000000 7FFFFF.FF -1 255 -00:00:00.01
|
||||||
|
* FFFFFFFFFF.FFD8F0 7FFFFF.9D -1 99 -00:00:00.99
|
||||||
|
* FFFFFFFFFF.F0E4D0 7FFFFF.00 -1 0 -00:00:01.00
|
||||||
|
* FFFFFFFFFF.000000 7FFFFE.FF -1 255 -00:00:01.01
|
||||||
|
* FFFFFFFFFE.FFD8F0 7FFFFE.F6 -2 246 -00:00:01.10
|
||||||
|
* FFFFFFFFFE.FE7960 Formula to convert fractional
|
||||||
|
* part from disk format (now stored in "frac"
|
||||||
|
* variable) to absolute value: "0x100 - frac". To
|
||||||
|
* reconstruct in-memory value, we shift to the next
|
||||||
|
* integer value and then substruct fractional part.
|
||||||
|
*/
|
||||||
|
intpart++; /* Shift to the next integer value */
|
||||||
|
frac -= 0x100; /* -(0x100 - frac) */
|
||||||
|
// fraclong = frac * 10000;
|
||||||
|
}
|
||||||
|
ltime = intpart << 24 + frac * 10000;
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
case 4:
|
||||||
|
intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
|
||||||
|
frac = buffer.getBeUint16();
|
||||||
|
if (intpart < 0 && frac > 0) {
|
||||||
|
/*
|
||||||
|
* Fix reverse fractional part order:
|
||||||
|
* "0x10000 - frac". See comments for FSP=1 and
|
||||||
|
* FSP=2 above.
|
||||||
|
*/
|
||||||
|
intpart++; /* Shift to the next integer value */
|
||||||
|
frac -= 0x10000; /* -(0x10000-frac) */
|
||||||
|
// fraclong = frac * 100;
|
||||||
|
}
|
||||||
|
ltime = intpart << 24 + frac * 100;
|
||||||
|
break;
|
||||||
|
case 5:
|
||||||
|
case 6:
|
||||||
|
intpart = buffer.getBeUlong48() - TIMEF_INT_OFS;
|
||||||
|
ltime = intpart;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
|
||||||
|
ltime = intpart << 24;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (intpart == 0) {
|
||||||
|
value = "00:00:00";
|
||||||
|
} else {
|
||||||
|
// 目前只记录秒，不处理us frac
|
||||||
|
// if (cal == null) cal = Calendar.getInstance();
|
||||||
|
// cal.clear();
|
||||||
|
// cal.set(70, 0, 1, (int) ((intpart >> 12) % (1 << 10)),
|
||||||
|
// (int) ((intpart >> 6) % (1 << 6)),
|
||||||
|
// (int) (intpart % (1 << 6)));
|
||||||
|
// value = new Time(cal.getTimeInMillis());
|
||||||
|
long ultime = Math.abs(ltime);
|
||||||
|
intpart = ultime >> 24;
|
||||||
|
value = String.format("%s%02d:%02d:%02d",
|
||||||
|
ltime >= 0 ? "" : "-",
|
||||||
|
(int) ((intpart >> 12) % (1 << 10)),
|
||||||
|
(int) ((intpart >> 6) % (1 << 6)),
|
||||||
|
(int) (intpart % (1 << 6)));
|
||||||
|
}
|
||||||
|
|
||||||
|
javaType = Types.TIME;
|
||||||
|
length = 3 + (meta + 1) / 2;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_NEWDATE: {
|
||||||
|
/*
|
||||||
|
* log_event.h : This enumeration value is only used internally
|
||||||
|
* and cannot exist in a binlog.
|
||||||
|
*/
|
||||||
|
logger.warn("MYSQL_TYPE_NEWDATE : This enumeration value is "
|
||||||
|
+ "only used internally and cannot exist in a binlog!");
|
||||||
|
javaType = Types.DATE;
|
||||||
|
value = null; /* unknown format */
|
||||||
|
length = 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_DATE: {
|
||||||
|
// MYSQL DataTypes:
|
||||||
|
// range: 0000-00-00 ~ 9999-12-31
|
||||||
|
final int i32 = buffer.getUint24();
|
||||||
|
if (i32 == 0) {
|
||||||
|
value = "0000-00-00";
|
||||||
|
} else {
|
||||||
|
// if (cal == null) cal = Calendar.getInstance();
|
||||||
|
// cal.clear();
|
||||||
|
/* month is 0-based, 0 for january. */
|
||||||
|
// cal.set((i32 / (16 * 32)), (i32 / 32 % 16) - 1, (i32 %
|
||||||
|
// 32));
|
||||||
|
// value = new java.sql.Date(cal.getTimeInMillis());
|
||||||
|
value = String.format("%04d-%02d-%02d", i32 / (16 * 32), i32 / 32 % 16, i32 % 32);
|
||||||
|
}
|
||||||
|
javaType = Types.DATE;
|
||||||
|
length = 3;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_YEAR: {
|
||||||
|
// MYSQL DataTypes: YEAR[(2|4)]
|
||||||
|
// In four-digit format, values display as 1901 to 2155, and
|
||||||
|
// 0000.
|
||||||
|
// In two-digit format, values display as 70 to 69, representing
|
||||||
|
// years from 1970 to 2069.
|
||||||
|
|
||||||
|
final int i32 = buffer.getUint8();
|
||||||
|
// If connection property 'YearIsDateType' has
|
||||||
|
// set, value is java.sql.Date.
|
||||||
|
/*
|
||||||
|
* if (cal == null) cal = Calendar.getInstance(); cal.clear();
|
||||||
|
* cal.set(Calendar.YEAR, i32 + 1900); value = new
|
||||||
|
* java.sql.Date(cal.getTimeInMillis());
|
||||||
|
*/
|
||||||
|
// The else, value is java.lang.Short.
|
||||||
|
if (i32 == 0) {
|
||||||
|
value = "0000";
|
||||||
|
} else {
|
||||||
|
value = String.valueOf((short) (i32 + 1900));
|
||||||
|
}
|
||||||
|
// It might seem more correct to create a java.sql.Types.DATE
|
||||||
|
// value
|
||||||
|
// for this date, but it is much simpler to pass the value as an
|
||||||
|
// integer. The MySQL JDBC specification states that one can
|
||||||
|
// pass a java int between 1901 and 2055. Creating a DATE value
|
||||||
|
// causes truncation errors with certain SQL_MODES
|
||||||
|
// (e.g."STRICT_TRANS_TABLES").
|
||||||
|
javaType = Types.VARCHAR; // Types.INTEGER;
|
||||||
|
length = 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_ENUM: {
|
||||||
|
final int int32;
|
||||||
|
/*
|
||||||
|
* log_event.h : This enumeration value is only used internally
|
||||||
|
* and cannot exist in a binlog.
|
||||||
|
*/
|
||||||
|
switch (len) {
|
||||||
|
case 1:
|
||||||
|
int32 = buffer.getUint8();
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
int32 = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("!! Unknown ENUM packlen = " + len);
|
||||||
|
}
|
||||||
|
// logger.warn("MYSQL_TYPE_ENUM : This enumeration value is "
|
||||||
|
// + "only used internally and cannot exist in a binlog!");
|
||||||
|
value = Integer.valueOf(int32);
|
||||||
|
javaType = Types.INTEGER;
|
||||||
|
length = len;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_SET: {
|
||||||
|
final int nbits = (meta & 0xFF) * 8;
|
||||||
|
len = (nbits + 7) / 8;
|
||||||
|
if (nbits > 1) {
|
||||||
|
// byte[] bits = new byte[len];
|
||||||
|
// buffer.fillBytes(bits, 0, len);
|
||||||
|
// 转化为unsign long
|
||||||
|
switch (len) {
|
||||||
|
case 1:
|
||||||
|
value = buffer.getInt8();
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
value = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
value = buffer.getUint24();
|
||||||
|
break;
|
||||||
|
case 4:
|
||||||
|
value = buffer.getUint32();
|
||||||
|
break;
|
||||||
|
case 5:
|
||||||
|
value = buffer.getUlong40();
|
||||||
|
break;
|
||||||
|
case 6:
|
||||||
|
value = buffer.getUlong48();
|
||||||
|
break;
|
||||||
|
case 7:
|
||||||
|
value = buffer.getUlong56();
|
||||||
|
break;
|
||||||
|
case 8:
|
||||||
|
value = buffer.getUlong64();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("!! Unknown Set len = " + len);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
final int bit = buffer.getInt8();
|
||||||
|
// value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE;
|
||||||
|
value = bit;
|
||||||
|
}
|
||||||
|
|
||||||
|
javaType = Types.BIT;
|
||||||
|
length = len;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_TINY_BLOB: {
|
||||||
|
/*
|
||||||
|
* log_event.h : This enumeration value is only used internally
|
||||||
|
* and cannot exist in a binlog.
|
||||||
|
*/
|
||||||
|
logger.warn("MYSQL_TYPE_TINY_BLOB : This enumeration value is "
|
||||||
|
+ "only used internally and cannot exist in a binlog!");
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_MEDIUM_BLOB: {
|
||||||
|
/*
|
||||||
|
* log_event.h : This enumeration value is only used internally
|
||||||
|
* and cannot exist in a binlog.
|
||||||
|
*/
|
||||||
|
logger.warn("MYSQL_TYPE_MEDIUM_BLOB : This enumeration value is "
|
||||||
|
+ "only used internally and cannot exist in a binlog!");
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_LONG_BLOB: {
|
||||||
|
/*
|
||||||
|
* log_event.h : This enumeration value is only used internally
|
||||||
|
* and cannot exist in a binlog.
|
||||||
|
*/
|
||||||
|
logger.warn("MYSQL_TYPE_LONG_BLOB : This enumeration value is "
|
||||||
|
+ "only used internally and cannot exist in a binlog!");
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_BLOB: {
|
||||||
|
/*
|
||||||
|
* BLOB or TEXT datatype
|
||||||
|
*/
|
||||||
|
switch (meta) {
|
||||||
|
case 1: {
|
||||||
|
/* TINYBLOB/TINYTEXT */
|
||||||
|
final int len8 = buffer.getUint8();
|
||||||
|
byte[] binary = new byte[len8];
|
||||||
|
buffer.fillBytes(binary, 0, len8);
|
||||||
|
value = binary;
|
||||||
|
javaType = Types.VARBINARY;
|
||||||
|
length = len8;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 2: {
|
||||||
|
/* BLOB/TEXT */
|
||||||
|
final int len16 = buffer.getUint16();
|
||||||
|
byte[] binary = new byte[len16];
|
||||||
|
buffer.fillBytes(binary, 0, len16);
|
||||||
|
value = binary;
|
||||||
|
javaType = Types.LONGVARBINARY;
|
||||||
|
length = len16;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 3: {
|
||||||
|
/* MEDIUMBLOB/MEDIUMTEXT */
|
||||||
|
final int len24 = buffer.getUint24();
|
||||||
|
byte[] binary = new byte[len24];
|
||||||
|
buffer.fillBytes(binary, 0, len24);
|
||||||
|
value = binary;
|
||||||
|
javaType = Types.LONGVARBINARY;
|
||||||
|
length = len24;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 4: {
|
||||||
|
/* LONGBLOB/LONGTEXT */
|
||||||
|
final int len32 = (int) buffer.getUint32();
|
||||||
|
byte[] binary = new byte[len32];
|
||||||
|
buffer.fillBytes(binary, 0, len32);
|
||||||
|
value = binary;
|
||||||
|
javaType = Types.LONGVARBINARY;
|
||||||
|
length = len32;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("!! Unknown BLOB packlen = " + meta);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_VARCHAR:
|
||||||
|
case LogEvent.MYSQL_TYPE_VAR_STRING: {
|
||||||
|
/*
|
||||||
|
* Except for the data length calculation, MYSQL_TYPE_VARCHAR,
|
||||||
|
* MYSQL_TYPE_VAR_STRING and MYSQL_TYPE_STRING are handled the
|
||||||
|
* same way.
|
||||||
|
*/
|
||||||
|
len = meta;
|
||||||
|
if (len < 256) {
|
||||||
|
len = buffer.getUint8();
|
||||||
|
} else {
|
||||||
|
len = buffer.getUint16();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isBinary) {
|
||||||
|
// fixed issue #66 ,binary类型在binlog中为var_string
|
||||||
|
/* fill binary */
|
||||||
|
byte[] binary = new byte[len];
|
||||||
|
buffer.fillBytes(binary, 0, len);
|
||||||
|
|
||||||
|
javaType = Types.VARBINARY;
|
||||||
|
value = binary;
|
||||||
|
} else {
|
||||||
|
value = buffer.getFullString(len, charsetName);
|
||||||
|
javaType = Types.VARCHAR;
|
||||||
|
}
|
||||||
|
|
||||||
|
length = len;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_STRING: {
|
||||||
|
if (len < 256) {
|
||||||
|
len = buffer.getUint8();
|
||||||
|
} else {
|
||||||
|
len = buffer.getUint16();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isBinary) {
|
||||||
|
/* fill binary */
|
||||||
|
byte[] binary = new byte[len];
|
||||||
|
buffer.fillBytes(binary, 0, len);
|
||||||
|
|
||||||
|
javaType = Types.BINARY;
|
||||||
|
value = binary;
|
||||||
|
} else {
|
||||||
|
value = buffer.getFullString(len, charsetName);
|
||||||
|
javaType = Types.CHAR; // Types.VARCHAR;
|
||||||
|
}
|
||||||
|
length = len;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case LogEvent.MYSQL_TYPE_GEOMETRY: {
|
||||||
|
/*
|
||||||
|
* MYSQL_TYPE_GEOMETRY: copy from BLOB or TEXT
|
||||||
|
*/
|
||||||
|
switch (meta) {
|
||||||
|
case 1:
|
||||||
|
len = buffer.getUint8();
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
len = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
case 3:
|
||||||
|
len = buffer.getUint24();
|
||||||
|
break;
|
||||||
|
case 4:
|
||||||
|
len = (int) buffer.getUint32();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new IllegalArgumentException("!! Unknown MYSQL_TYPE_GEOMETRY packlen = " + meta);
|
||||||
|
}
|
||||||
|
/* fill binary */
|
||||||
|
byte[] binary = new byte[len];
|
||||||
|
buffer.fillBytes(binary, 0, len);
|
||||||
|
|
||||||
|
/* Warning unsupport cloumn type */
|
||||||
|
logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d",
|
||||||
|
meta,
|
||||||
|
meta,
|
||||||
|
len));
|
||||||
|
javaType = Types.BINARY;
|
||||||
|
value = binary;
|
||||||
|
length = len;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
logger.error(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)",
|
||||||
|
type,
|
||||||
|
meta,
|
||||||
|
meta));
|
||||||
|
javaType = Types.OTHER;
|
||||||
|
value = null;
|
||||||
|
length = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final boolean isNull() {
|
||||||
|
return fNull;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getJavaType() {
|
||||||
|
return javaType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final Serializable getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getLength() {
|
||||||
|
return length;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.util.BitSet;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogContext;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Common base class for all row-containing log events.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public abstract class RowsLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Fixed data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>6 bytes. The table ID.</li>
|
||||||
|
* <li>2 bytes. Reserved for future use.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Variable data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>Packed integer. The number of columns in the table.</li>
|
||||||
|
*
|
||||||
|
* <li>Variable-sized. Bit-field indicating whether each column is used, one
|
||||||
|
* bit per column. For this field, the amount of storage required for N
|
||||||
|
* columns is INT((N+7)/8) bytes.</li>
|
||||||
|
*
|
||||||
|
* <li>Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating
|
||||||
|
* whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one
|
||||||
|
* bit per column. For this field, the amount of storage required for N
|
||||||
|
* columns is INT((N+7)/8) bytes.</li>
|
||||||
|
*
|
||||||
|
* <li>Variable-sized. A sequence of zero or more rows. The end is
|
||||||
|
* determined by the size of the event. Each row has the following format:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>Variable-sized. Bit-field indicating whether each field in the row is
|
||||||
|
* NULL. Only columns that are "used" according to the second field in the
|
||||||
|
* variable data part are listed here. If the second field in the variable
|
||||||
|
* data part has N one-bits, the amount of storage required for this field
|
||||||
|
* is INT((N+7)/8) bytes.</li>
|
||||||
|
*
|
||||||
|
* <li>Variable-sized. The row-image, containing values of all table fields.
|
||||||
|
* This only lists table fields that are used (according to the second field
|
||||||
|
* of the variable data part) and non-NULL (according to the previous
|
||||||
|
* field). In other words, the number of values listed here is equal to the
|
||||||
|
* number of zero bits in the previous field (not counting padding bits in
|
||||||
|
* the last byte). The format of each value is described in the
|
||||||
|
* log_event_print_value() function in log_event.cc.</li>
|
||||||
|
*
|
||||||
|
* <li>(for UPDATE_ROWS_EVENT only) the previous two fields are repeated,
|
||||||
|
* representing a second table row.</li>
|
||||||
|
* </ul>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
private final long tableId; /* Table ID */
|
||||||
|
private TableMapLogEvent table; /* The table the rows belong to */
|
||||||
|
|
||||||
|
/** Bitmap denoting columns available */
|
||||||
|
protected final int columnLen;
|
||||||
|
protected final BitSet columns;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bitmap for columns available in the after image, if present. These fields
|
||||||
|
* are only available for Update_rows events. Observe that the width of both
|
||||||
|
* the before image COLS vector and the after image COLS vector is the same:
|
||||||
|
* the number of columns of the table on the master.
|
||||||
|
*/
|
||||||
|
protected final BitSet changeColumns;
|
||||||
|
|
||||||
|
/** XXX: Don't handle buffer in another thread. */
|
||||||
|
private final LogBuffer rowsBuf; /* The rows in packed format */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* enum enum_flag
|
||||||
|
*
|
||||||
|
* These definitions allow you to combine the flags into an appropriate flag
|
||||||
|
* set using the normal bitwise operators. The implicit conversion from an
|
||||||
|
* enum-constant to an integer is accepted by the compiler, which is then
|
||||||
|
* used to set the real set of flags.
|
||||||
|
*/
|
||||||
|
private final int flags;
|
||||||
|
|
||||||
|
/** Last event of a statement */
|
||||||
|
public static final int STMT_END_F = 1;
|
||||||
|
|
||||||
|
/** Value of the OPTION_NO_FOREIGN_KEY_CHECKS flag in thd->options */
|
||||||
|
public static final int NO_FOREIGN_KEY_CHECKS_F = (1 << 1);
|
||||||
|
|
||||||
|
/** Value of the OPTION_RELAXED_UNIQUE_CHECKS flag in thd->options */
|
||||||
|
public static final int RELAXED_UNIQUE_CHECKS_F = (1 << 2);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Indicates that rows in this event are complete, that is contain values
|
||||||
|
* for all columns of the table.
|
||||||
|
*/
|
||||||
|
public static final int COMPLETE_ROWS_F = (1 << 3);
|
||||||
|
|
||||||
|
/* RW = "RoWs" */
|
||||||
|
public static final int RW_MAPID_OFFSET = 0;
|
||||||
|
public static final int RW_FLAGS_OFFSET = 6;
|
||||||
|
public static final int RW_VHLEN_OFFSET = 8;
|
||||||
|
public static final int RW_V_TAG_LEN = 1;
|
||||||
|
public static final int RW_V_EXTRAINFO_TAG = 0;
|
||||||
|
|
||||||
|
|
||||||
|
public RowsLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
|
||||||
|
int headerLen = 0;
|
||||||
|
buffer.position(commonHeaderLen + RW_MAPID_OFFSET);
|
||||||
|
if (postHeaderLen == 6)
|
||||||
|
{
|
||||||
|
/* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
|
||||||
|
tableId = buffer.getUint32();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET
|
||||||
|
}
|
||||||
|
flags = buffer.getUint16();
|
||||||
|
|
||||||
|
if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2)
|
||||||
|
{
|
||||||
|
headerLen = buffer.getUint16();
|
||||||
|
headerLen -= 2;
|
||||||
|
int start = buffer.position();
|
||||||
|
int end = start + headerLen;
|
||||||
|
for(int i = start ;i < end; ){
|
||||||
|
switch (buffer.getUint8(i++)) {
|
||||||
|
case RW_V_EXTRAINFO_TAG:
|
||||||
|
// int infoLen = buffer.getUint8();
|
||||||
|
buffer.position(i + EXTRA_ROW_INFO_LEN_OFFSET);
|
||||||
|
int checkLen = buffer.getUint8(); // EXTRA_ROW_INFO_LEN_OFFSET
|
||||||
|
int val= checkLen - EXTRA_ROW_INFO_HDR_BYTES;
|
||||||
|
assert(buffer.getUint8() == val); //EXTRA_ROW_INFO_FORMAT_OFFSET
|
||||||
|
for (int j= 0; j < val; j++) {
|
||||||
|
assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
i = end;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer.position(commonHeaderLen + postHeaderLen + headerLen);
|
||||||
|
columnLen = (int) buffer.getPackedLong();
|
||||||
|
columns = buffer.getBitmap(columnLen);
|
||||||
|
|
||||||
|
if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT)
|
||||||
|
{
|
||||||
|
changeColumns = buffer.getBitmap(columnLen);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
changeColumns = columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
// XXX: Don't handle buffer in another thread.
|
||||||
|
int dataSize = buffer.limit() - buffer.position();
|
||||||
|
rowsBuf = buffer.duplicate(dataSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final void fillTable(LogContext context)
|
||||||
|
{
|
||||||
|
table = context.getTable(tableId);
|
||||||
|
|
||||||
|
// end of statement check:
|
||||||
|
if ((flags & RowsLogEvent.STMT_END_F) != 0)
|
||||||
|
{
|
||||||
|
// Now is safe to clear ignored map (clear_tables will also
|
||||||
|
// delete original table map events stored in the map).
|
||||||
|
context.clearAllTables();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getTableId()
|
||||||
|
{
|
||||||
|
return tableId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final TableMapLogEvent getTable()
|
||||||
|
{
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final BitSet getColumns()
|
||||||
|
{
|
||||||
|
return columns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final BitSet getChangeColumns()
|
||||||
|
{
|
||||||
|
return changeColumns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final RowsLogBuffer getRowsBuf(String charsetName)
|
||||||
|
{
|
||||||
|
return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getFlags(final int flags)
|
||||||
|
{
|
||||||
|
return this.flags & flags;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author jianghang 2013-4-8 上午12:36:29
|
||||||
|
* @version 1.0.3
|
||||||
|
* @since mysql 5.6
|
||||||
|
*/
|
||||||
|
public class RowsQueryLogEvent extends IgnorableLogEvent {
|
||||||
|
|
||||||
|
private String rowsQuery;
|
||||||
|
|
||||||
|
public RowsQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
|
||||||
|
|
||||||
|
/*
|
||||||
|
* m_rows_query length is stored using only one byte, but that length is
|
||||||
|
* ignored and the complete query is read.
|
||||||
|
*/
|
||||||
|
int offset = commonHeaderLen + postHeaderLen + 1;
|
||||||
|
int len = buffer.limit() - offset;
|
||||||
|
rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRowsQuery() {
|
||||||
|
return rowsQuery;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and
|
||||||
|
* 4.x).
|
||||||
|
*
|
||||||
|
* Format_description_log_event derives from Start_log_event_v3; it is the
|
||||||
|
* Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that
|
||||||
|
* describes the other events' Common-Header/Post-Header lengths. This event is
|
||||||
|
* sent by MySQL 5.0 whenever it starts sending a new binlog if the requested
|
||||||
|
* position is >4 (otherwise if ==4 the event will be sent naturally).
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public class StartLogEventV3 extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* We could have used SERVER_VERSION_LENGTH, but this introduces an obscure
|
||||||
|
* dependency - if somebody decided to change SERVER_VERSION_LENGTH this
|
||||||
|
* would break the replication protocol
|
||||||
|
*/
|
||||||
|
public static final int ST_SERVER_VER_LEN = 50;
|
||||||
|
|
||||||
|
/* start event post-header (for v3 and v4) */
|
||||||
|
public static final int ST_BINLOG_VER_OFFSET = 0;
|
||||||
|
public static final int ST_SERVER_VER_OFFSET = 2;
|
||||||
|
|
||||||
|
protected int binlogVersion;
|
||||||
|
protected String serverVersion;
|
||||||
|
|
||||||
|
public StartLogEventV3(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen);
|
||||||
|
binlogVersion = buffer.getUint16(); // ST_BINLOG_VER_OFFSET
|
||||||
|
serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
|
||||||
|
}
|
||||||
|
|
||||||
|
public StartLogEventV3()
|
||||||
|
{
|
||||||
|
super(new LogHeader(START_EVENT_V3));
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getServerVersion()
|
||||||
|
{
|
||||||
|
return serverVersion;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getBinlogVersion()
|
||||||
|
{
|
||||||
|
return binlogVersion;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stop_log_event.
|
||||||
|
*
|
||||||
|
* The Post-Header and Body for this event type are empty; it only has the
|
||||||
|
* Common-Header.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class StopLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
public StopLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent description_event)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,543 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.util.BitSet;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* In row-based mode, every row operation event is preceded by a
|
||||||
|
* Table_map_log_event which maps a table definition to a number. The table
|
||||||
|
* definition consists of database name, table name, and column definitions.
|
||||||
|
*
|
||||||
|
* The Post-Header has the following components:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Post-Header for Table_map_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>table_id</td>
|
||||||
|
* <td>6 bytes unsigned integer</td>
|
||||||
|
* <td>The number that identifies the table.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>flags</td>
|
||||||
|
* <td>2 byte bitfield</td>
|
||||||
|
* <td>Reserved for future use; currently always 0.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* The Body has the following components:
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Body for Table_map_log_event</caption>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Format</th>
|
||||||
|
* <th>Description</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>database_name</td>
|
||||||
|
* <td>one byte string length, followed by null-terminated string</td>
|
||||||
|
* <td>The name of the database in which the table resides. The name is
|
||||||
|
* represented as a one byte unsigned integer representing the number of bytes
|
||||||
|
* in the name, followed by length bytes containing the database name, followed
|
||||||
|
* by a terminating 0 byte. (Note the redundancy in the representation of the
|
||||||
|
* length.)</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>table_name</td>
|
||||||
|
* <td>one byte string length, followed by null-terminated string</td>
|
||||||
|
* <td>The name of the table, encoded the same way as the database name above.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>column_count</td>
|
||||||
|
* <td>packed_integer "Packed Integer"</td>
|
||||||
|
* <td>The number of columns in the table, represented as a packed
|
||||||
|
* variable-length integer.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>column_type</td>
|
||||||
|
* <td>List of column_count 1 byte enumeration values</td>
|
||||||
|
* <td>The type of each column in the table, listed from left to right. Each
|
||||||
|
* byte is mapped to a column type according to the enumeration type
|
||||||
|
* enum_field_types defined in mysql_com.h. The mapping of types to numbers is
|
||||||
|
* listed in the table Table_table_map_log_event_column_types "below" (along
|
||||||
|
* with description of the associated metadata field).</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>metadata_length</td>
|
||||||
|
* <td>packed_integer "Packed Integer"</td>
|
||||||
|
* <td>The length of the following metadata block</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>metadata</td>
|
||||||
|
* <td>list of metadata for each column</td>
|
||||||
|
* <td>For each column from left to right, a chunk of data who's length and
|
||||||
|
* semantics depends on the type of the column. The length and semantics for the
|
||||||
|
* metadata for each column are listed in the table
|
||||||
|
* Table_table_map_log_event_column_types "below".</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>null_bits</td>
|
||||||
|
* <td>column_count bits, rounded up to nearest byte</td>
|
||||||
|
* <td>For each column, a bit indicating whether data in the column can be NULL
|
||||||
|
* or not. The number of bytes needed for this is int((column_count+7)/8). The
|
||||||
|
* flag for the first column from the left is in the least-significant bit of
|
||||||
|
* the first byte, the second is in the second least significant bit of the
|
||||||
|
* first byte, the ninth is in the least significant bit of the second byte, and
|
||||||
|
* so on.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* The table below lists all column types, along with the numerical identifier
|
||||||
|
* for it and the size and interpretation of meta-data used to describe the
|
||||||
|
* type.
|
||||||
|
*
|
||||||
|
* <table>
|
||||||
|
* <caption>Table_map_log_event column types: numerical identifier and
|
||||||
|
* metadata</caption>
|
||||||
|
* <tr>
|
||||||
|
* <th>Name</th>
|
||||||
|
* <th>Identifier</th>
|
||||||
|
* <th>Size of metadata in bytes</th>
|
||||||
|
* <th>Description of metadata</th>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_DECIMAL</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_TINY</td>
|
||||||
|
* <td>1</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_SHORT</td>
|
||||||
|
* <td>2</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_LONG</td>
|
||||||
|
* <td>3</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_FLOAT</td>
|
||||||
|
* <td>4</td>
|
||||||
|
* <td>1 byte</td>
|
||||||
|
* <td>1 byte unsigned integer, representing the "pack_length", which is equal
|
||||||
|
* to sizeof(float) on the server from which the event originates.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_DOUBLE</td>
|
||||||
|
* <td>5</td>
|
||||||
|
* <td>1 byte</td>
|
||||||
|
* <td>1 byte unsigned integer, representing the "pack_length", which is equal
|
||||||
|
* to sizeof(double) on the server from which the event originates.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_NULL</td>
|
||||||
|
* <td>6</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_TIMESTAMP</td>
|
||||||
|
* <td>7</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_LONGLONG</td>
|
||||||
|
* <td>8</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_INT24</td>
|
||||||
|
* <td>9</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_DATE</td>
|
||||||
|
* <td>10</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_TIME</td>
|
||||||
|
* <td>11</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_DATETIME</td>
|
||||||
|
* <td>12</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_YEAR</td>
|
||||||
|
* <td>13</td>
|
||||||
|
* <td>0</td>
|
||||||
|
* <td>No column metadata.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td><i>MYSQL_TYPE_NEWDATE</i></td>
|
||||||
|
* <td><i>14</i></td>
|
||||||
|
* <td>–</td>
|
||||||
|
* <td><i>This enumeration value is only used internally and cannot exist in a
|
||||||
|
* binlog.</i></td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_VARCHAR</td>
|
||||||
|
* <td>15</td>
|
||||||
|
* <td>2 bytes</td>
|
||||||
|
* <td>2 byte unsigned integer representing the maximum length of the string.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_BIT</td>
|
||||||
|
* <td>16</td>
|
||||||
|
* <td>2 bytes</td>
|
||||||
|
* <td>A 1 byte unsigned int representing the length in bits of the bitfield (0
|
||||||
|
* to 64), followed by a 1 byte unsigned int representing the number of bytes
|
||||||
|
* occupied by the bitfield. The number of bytes is either int((length+7)/8) or
|
||||||
|
* int(length/8).</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_NEWDECIMAL</td>
|
||||||
|
* <td>246</td>
|
||||||
|
* <td>2 bytes</td>
|
||||||
|
* <td>A 1 byte unsigned int representing the precision, followed by a 1 byte
|
||||||
|
* unsigned int representing the number of decimals.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td><i>MYSQL_TYPE_ENUM</i></td>
|
||||||
|
* <td><i>247</i></td>
|
||||||
|
* <td>–</td>
|
||||||
|
* <td><i>This enumeration value is only used internally and cannot exist in a
|
||||||
|
* binlog.</i></td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td><i>MYSQL_TYPE_SET</i></td>
|
||||||
|
* <td><i>248</i></td>
|
||||||
|
* <td>–</td>
|
||||||
|
* <td><i>This enumeration value is only used internally and cannot exist in a
|
||||||
|
* binlog.</i></td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_TINY_BLOB</td>
|
||||||
|
* <td>249</td>
|
||||||
|
* <td>–</td>
|
||||||
|
* <td><i>This enumeration value is only used internally and cannot exist in a
|
||||||
|
* binlog.</i></td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td><i>MYSQL_TYPE_MEDIUM_BLOB</i></td>
|
||||||
|
* <td><i>250</i></td>
|
||||||
|
* <td>–</td>
|
||||||
|
* <td><i>This enumeration value is only used internally and cannot exist in a
|
||||||
|
* binlog.</i></td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td><i>MYSQL_TYPE_LONG_BLOB</i></td>
|
||||||
|
* <td><i>251</i></td>
|
||||||
|
* <td>–</td>
|
||||||
|
* <td><i>This enumeration value is only used internally and cannot exist in a
|
||||||
|
* binlog.</i></td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_BLOB</td>
|
||||||
|
* <td>252</td>
|
||||||
|
* <td>1 byte</td>
|
||||||
|
* <td>The pack length, i.e., the number of bytes needed to represent the length
|
||||||
|
* of the blob: 1, 2, 3, or 4.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_VAR_STRING</td>
|
||||||
|
* <td>253</td>
|
||||||
|
* <td>2 bytes</td>
|
||||||
|
* <td>This is used to store both strings and enumeration values. The first byte
|
||||||
|
* is a enumeration value storing the <i>real type</i>, which may be either
|
||||||
|
* MYSQL_TYPE_VAR_STRING or MYSQL_TYPE_ENUM. The second byte is a 1 byte
|
||||||
|
* unsigned integer representing the field size, i.e., the number of bytes
|
||||||
|
* needed to store the length of the string.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_STRING</td>
|
||||||
|
* <td>254</td>
|
||||||
|
* <td>2 bytes</td>
|
||||||
|
* <td>The first byte is always MYSQL_TYPE_VAR_STRING (i.e., 253). The second
|
||||||
|
* byte is the field size, i.e., the number of bytes in the representation of
|
||||||
|
* size of the string: 3 or 4.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* <tr>
|
||||||
|
* <td>MYSQL_TYPE_GEOMETRY</td>
|
||||||
|
* <td>255</td>
|
||||||
|
* <td>1 byte</td>
|
||||||
|
* <td>The pack length, i.e., the number of bytes needed to represent the length
|
||||||
|
* of the geometry: 1, 2, 3, or 4.</td>
|
||||||
|
* </tr>
|
||||||
|
*
|
||||||
|
* </table>
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class TableMapLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Fixed data part:
|
||||||
|
* <ul>
|
||||||
|
* <li>6 bytes. The table ID.</li>
|
||||||
|
* <li>2 bytes. Reserved for future use.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Variable data part:
|
||||||
|
* <ul>
|
||||||
|
* <li>1 byte. The length of the database name.</li>
|
||||||
|
* <li>Variable-sized. The database name (null-terminated).</li>
|
||||||
|
* <li>1 byte. The length of the table name.</li>
|
||||||
|
* <li>Variable-sized. The table name (null-terminated).</li>
|
||||||
|
* <li>Packed integer. The number of columns in the table.</li>
|
||||||
|
* <li>Variable-sized. An array of column types, one byte per column.</li>
|
||||||
|
* <li>Packed integer. The length of the metadata block.</li>
|
||||||
|
* <li>Variable-sized. The metadata block; see log_event.h for contents and
|
||||||
|
* format.</li>
|
||||||
|
* <li>Variable-sized. Bit-field indicating whether each column can be NULL,
|
||||||
|
* one bit per column. For this field, the amount of storage required for N
|
||||||
|
* columns is INT((N+7)/8) bytes.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
protected final String dbname;
|
||||||
|
protected final String tblname;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Holding mysql column information.
|
||||||
|
*/
|
||||||
|
public static final class ColumnInfo
|
||||||
|
{
|
||||||
|
public int type;
|
||||||
|
public int meta;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected final int columnCnt;
|
||||||
|
protected final ColumnInfo[] columnInfo; // buffer for field metadata
|
||||||
|
|
||||||
|
protected final long tableId;
|
||||||
|
protected BitSet nullBits;
|
||||||
|
|
||||||
|
/** TM = "Table Map" */
|
||||||
|
public static final int TM_MAPID_OFFSET = 0;
|
||||||
|
public static final int TM_FLAGS_OFFSET = 6;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor used by slave to read the event from the binary log.
|
||||||
|
*/
|
||||||
|
public TableMapLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.commonHeaderLen;
|
||||||
|
final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
|
||||||
|
/* Read the post-header */
|
||||||
|
buffer.position(commonHeaderLen + TM_MAPID_OFFSET);
|
||||||
|
if (postHeaderLen == 6)
|
||||||
|
{
|
||||||
|
/* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
|
||||||
|
tableId = buffer.getUint32();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN);
|
||||||
|
tableId = buffer.getUlong48();
|
||||||
|
}
|
||||||
|
// flags = buffer.getUint16();
|
||||||
|
|
||||||
|
/* Read the variable part of the event */
|
||||||
|
buffer.position(commonHeaderLen + postHeaderLen);
|
||||||
|
dbname = buffer.getString();
|
||||||
|
buffer.forward(1); /* termination null */
|
||||||
|
tblname = buffer.getString();
|
||||||
|
buffer.forward(1); /* termination null */
|
||||||
|
|
||||||
|
// Read column information from buffer
|
||||||
|
columnCnt = (int) buffer.getPackedLong();
|
||||||
|
columnInfo = new ColumnInfo[columnCnt];
|
||||||
|
for (int i = 0; i < columnCnt; i++)
|
||||||
|
{
|
||||||
|
ColumnInfo info = new ColumnInfo();
|
||||||
|
info.type = buffer.getUint8();
|
||||||
|
columnInfo[i] = info;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (buffer.position() < buffer.limit())
|
||||||
|
{
|
||||||
|
final int fieldSize = (int) buffer.getPackedLong();
|
||||||
|
decodeFields(buffer, fieldSize);
|
||||||
|
nullBits = buffer.getBitmap(columnCnt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decode field metadata by column types.
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60/sql/rpl_utility.h
|
||||||
|
*/
|
||||||
|
private final void decodeFields(LogBuffer buffer, final int len)
|
||||||
|
{
|
||||||
|
final int limit = buffer.limit();
|
||||||
|
|
||||||
|
buffer.limit(len + buffer.position());
|
||||||
|
for (int i = 0; i < columnCnt; i++)
|
||||||
|
{
|
||||||
|
ColumnInfo info = columnInfo[i];
|
||||||
|
|
||||||
|
switch (info.type)
|
||||||
|
{
|
||||||
|
case MYSQL_TYPE_TINY_BLOB:
|
||||||
|
case MYSQL_TYPE_BLOB:
|
||||||
|
case MYSQL_TYPE_MEDIUM_BLOB:
|
||||||
|
case MYSQL_TYPE_LONG_BLOB:
|
||||||
|
case MYSQL_TYPE_DOUBLE:
|
||||||
|
case MYSQL_TYPE_FLOAT:
|
||||||
|
case MYSQL_TYPE_GEOMETRY:
|
||||||
|
/*
|
||||||
|
These types store a single byte.
|
||||||
|
*/
|
||||||
|
info.meta = buffer.getUint8();
|
||||||
|
break;
|
||||||
|
case MYSQL_TYPE_SET:
|
||||||
|
case MYSQL_TYPE_ENUM:
|
||||||
|
/*
|
||||||
|
* log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This
|
||||||
|
* enumeration value is only used internally and cannot
|
||||||
|
* exist in a binlog.
|
||||||
|
*/
|
||||||
|
logger.warn("This enumeration value is only used internally "
|
||||||
|
+ "and cannot exist in a binlog: type=" + info.type);
|
||||||
|
break;
|
||||||
|
case MYSQL_TYPE_STRING:
|
||||||
|
{
|
||||||
|
/*
|
||||||
|
* log_event.h : The first byte is always
|
||||||
|
* MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the
|
||||||
|
* field size, i.e., the number of bytes in the
|
||||||
|
* representation of size of the string: 3 or 4.
|
||||||
|
*/
|
||||||
|
int x = (buffer.getUint8() << 8); // real_type
|
||||||
|
x += buffer.getUint8(); // pack or field length
|
||||||
|
info.meta = x;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case MYSQL_TYPE_BIT:
|
||||||
|
info.meta = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
case MYSQL_TYPE_VARCHAR:
|
||||||
|
/*
|
||||||
|
* These types store two bytes.
|
||||||
|
*/
|
||||||
|
info.meta = buffer.getUint16();
|
||||||
|
break;
|
||||||
|
case MYSQL_TYPE_NEWDECIMAL:
|
||||||
|
{
|
||||||
|
int x = buffer.getUint8() << 8; // precision
|
||||||
|
x += buffer.getUint8(); // decimals
|
||||||
|
info.meta = x;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case MYSQL_TYPE_TIME2:
|
||||||
|
case MYSQL_TYPE_DATETIME2:
|
||||||
|
case MYSQL_TYPE_TIMESTAMP2:
|
||||||
|
{
|
||||||
|
info.meta = buffer.getUint8();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
info.meta = 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buffer.limit(limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getDbName()
|
||||||
|
{
|
||||||
|
return dbname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getTableName()
|
||||||
|
{
|
||||||
|
return tblname;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final int getColumnCnt()
|
||||||
|
{
|
||||||
|
return columnCnt;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final ColumnInfo[] getColumnInfo()
|
||||||
|
{
|
||||||
|
return columnInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getTableId()
|
||||||
|
{
|
||||||
|
return tableId;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unknown_log_event
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class UnknownLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
public UnknownLogEvent(LogHeader header)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log row updates with a before image. The event contain several update rows
|
||||||
|
* for a table. Note that each event contains only rows for one table.
|
||||||
|
*
|
||||||
|
* Also note that the row data consists of pairs of row data: one row for the
|
||||||
|
* old data and one row for the new data.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class UpdateRowsLogEvent extends RowsLogEvent
|
||||||
|
{
|
||||||
|
public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.CharsetConversion;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* User_var_log_event.
|
||||||
|
*
|
||||||
|
* Every time a query uses the value of a user variable, a User_var_log_event is
|
||||||
|
* written before the Query_log_event, to set the user variable.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class UserVarLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Fixed data part: Empty
|
||||||
|
*
|
||||||
|
* <p>
|
||||||
|
* Variable data part:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>4 bytes. the size of the user variable name.</li>
|
||||||
|
* <li>The user variable name.</li>
|
||||||
|
* <li>1 byte. Non-zero if the variable value is the SQL NULL value, 0
|
||||||
|
* otherwise. If this byte is 0, the following parts exist in the event.</li>
|
||||||
|
* <li>1 byte. The user variable type. The value corresponds to elements of
|
||||||
|
* enum Item_result defined in include/mysql_com.h.</li>
|
||||||
|
* <li>4 bytes. The number of the character set for the user variable
|
||||||
|
* (needed for a string variable). The character set number is really a
|
||||||
|
* collation number that indicates a character set/collation pair.</li>
|
||||||
|
* <li>4 bytes. The size of the user variable value (corresponds to member
|
||||||
|
* val_len of class Item_string).</li>
|
||||||
|
* <li>Variable-sized. For a string variable, this is the string. For a
|
||||||
|
* float or integer variable, this is its value in 8 bytes.</li>
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
|
||||||
|
*/
|
||||||
|
private final String name;
|
||||||
|
private final Serializable value;
|
||||||
|
private final int type;
|
||||||
|
private final int charsetNumber;
|
||||||
|
private final boolean isNull;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The following is for user defined functions
|
||||||
|
*
|
||||||
|
* @see mysql-5.1.60//include/mysql_com.h
|
||||||
|
*/
|
||||||
|
public static final int STRING_RESULT = 0;
|
||||||
|
public static final int REAL_RESULT = 1;
|
||||||
|
public static final int INT_RESULT = 2;
|
||||||
|
public static final int ROW_RESULT = 3;
|
||||||
|
public static final int DECIMAL_RESULT = 4;
|
||||||
|
|
||||||
|
/* User_var event data */
|
||||||
|
public static final int UV_VAL_LEN_SIZE = 4;
|
||||||
|
public static final int UV_VAL_IS_NULL = 1;
|
||||||
|
public static final int UV_VAL_TYPE_SIZE = 1;
|
||||||
|
public static final int UV_NAME_LEN_SIZE = 4;
|
||||||
|
public static final int UV_CHARSET_NUMBER_SIZE = 4;
|
||||||
|
|
||||||
|
public UserVarLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent) throws IOException
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
/* The Post-Header is empty. The Variable Data part begins immediately. */
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen
|
||||||
|
+ descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
|
||||||
|
final int nameLen = (int) buffer.getUint32();
|
||||||
|
name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE
|
||||||
|
isNull = (0 != buffer.getInt8());
|
||||||
|
|
||||||
|
if (isNull)
|
||||||
|
{
|
||||||
|
type = STRING_RESULT;
|
||||||
|
charsetNumber = 63; /* binary */
|
||||||
|
value = null;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
type = buffer.getInt8(); // UV_VAL_IS_NULL
|
||||||
|
charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE
|
||||||
|
final int valueLen = (int) buffer.getUint32(); // buf + UV_CHARSET_NUMBER_SIZE
|
||||||
|
final int limit = buffer.limit(); /* for restore */
|
||||||
|
buffer.limit(buffer.position() + valueLen);
|
||||||
|
|
||||||
|
/* @see User_var_log_event::print */
|
||||||
|
switch (type)
|
||||||
|
{
|
||||||
|
case REAL_RESULT:
|
||||||
|
value = Double.valueOf(buffer.getDouble64()); // float8get
|
||||||
|
break;
|
||||||
|
case INT_RESULT:
|
||||||
|
if (valueLen == 8)
|
||||||
|
value = Long.valueOf(buffer.getLong64()); // !uint8korr
|
||||||
|
else if (valueLen == 4)
|
||||||
|
value = Long.valueOf(buffer.getUint32());
|
||||||
|
else
|
||||||
|
throw new IOException("Error INT_RESULT length: "
|
||||||
|
+ valueLen);
|
||||||
|
break;
|
||||||
|
case DECIMAL_RESULT:
|
||||||
|
final int precision = buffer.getInt8();
|
||||||
|
final int scale = buffer.getInt8();
|
||||||
|
value = buffer.getDecimal(precision, scale); // bin2decimal
|
||||||
|
break;
|
||||||
|
case STRING_RESULT:
|
||||||
|
String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
|
||||||
|
value = buffer.getFixString(valueLen, charsetName);
|
||||||
|
break;
|
||||||
|
case ROW_RESULT:
|
||||||
|
// this seems to be banned in MySQL altogether
|
||||||
|
throw new IOException("ROW_RESULT is unsupported");
|
||||||
|
default:
|
||||||
|
value = null;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
buffer.limit(limit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public final String getQuery()
|
||||||
|
{
|
||||||
|
if (value == null)
|
||||||
|
{
|
||||||
|
return "SET @" + name + " := NULL";
|
||||||
|
}
|
||||||
|
else if (type == STRING_RESULT)
|
||||||
|
{
|
||||||
|
// TODO: do escaping !?
|
||||||
|
return "SET @" + name + " := \'" + value + '\'';
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return "SET @" + name + " := " + String.valueOf(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Log row insertions and updates. The event contain several insert/update rows
|
||||||
|
* for a table. Note that each event contains only rows for one table.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class WriteRowsLogEvent extends RowsLogEvent
|
||||||
|
{
|
||||||
|
public WriteRowsLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogEvent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Logs xid of the transaction-to-be-committed in the 2pc protocol. Has no
|
||||||
|
* meaning in replication, slaves ignore it.
|
||||||
|
*
|
||||||
|
* @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
|
||||||
|
* @version 1.0
|
||||||
|
*/
|
||||||
|
public final class XidLogEvent extends LogEvent
|
||||||
|
{
|
||||||
|
private final long xid;
|
||||||
|
|
||||||
|
public XidLogEvent(LogHeader header, LogBuffer buffer,
|
||||||
|
FormatDescriptionLogEvent descriptionEvent)
|
||||||
|
{
|
||||||
|
super(header);
|
||||||
|
|
||||||
|
/* The Post-Header is empty. The Variable Data part begins immediately. */
|
||||||
|
buffer.position(descriptionEvent.commonHeaderLen
|
||||||
|
+ descriptionEvent.postHeaderLen[XID_EVENT - 1]);
|
||||||
|
xid = buffer.getLong64(); // !uint8korr
|
||||||
|
}
|
||||||
|
|
||||||
|
public final long getXid()
|
||||||
|
{
|
||||||
|
return xid;
|
||||||
|
}
|
||||||
|
}
|
||||||
+33
@@ -0,0 +1,33 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event.mariadb;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mariadb的ANNOTATE_ROWS_EVENT类型
|
||||||
|
*
|
||||||
|
* @author jianghang 2014-1-20 下午2:20:35
|
||||||
|
* @since 1.0.17
|
||||||
|
*/
|
||||||
|
public class AnnotateRowsEvent extends IgnorableLogEvent {
|
||||||
|
|
||||||
|
private String rowsQuery;
|
||||||
|
|
||||||
|
public AnnotateRowsEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
|
||||||
|
final int commonHeaderLen = descriptionEvent.getCommonHeaderLen();
|
||||||
|
final int postHeaderLen = descriptionEvent.getPostHeaderLen()[header.getType() - 1];
|
||||||
|
|
||||||
|
int offset = commonHeaderLen + postHeaderLen;
|
||||||
|
int len = buffer.limit() - offset;
|
||||||
|
rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRowsQuery() {
|
||||||
|
return rowsQuery;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event.mariadb;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mariadb10的BINLOG_CHECKPOINT_EVENT类型
|
||||||
|
*
|
||||||
|
* @author jianghang 2014-1-20 下午2:22:04
|
||||||
|
* @since 1.0.17
|
||||||
|
*/
|
||||||
|
public class BinlogCheckPointLogEvent extends IgnorableLogEvent {
|
||||||
|
|
||||||
|
public BinlogCheckPointLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
// do nothing , just mariadb binlog checkpoint
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event.mariadb;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mariadb的GTID_LIST_EVENT类型
|
||||||
|
*
|
||||||
|
* @author jianghang 2014-1-20 下午4:51:50
|
||||||
|
* @since 1.0.17
|
||||||
|
*/
|
||||||
|
public class MariaGtidListLogEvent extends IgnorableLogEvent {
|
||||||
|
|
||||||
|
public MariaGtidListLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
// do nothing , just ignore log event
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog.event.mariadb;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.LogHeader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mariadb的GTID_EVENT类型
|
||||||
|
*
|
||||||
|
* @author jianghang 2014-1-20 下午4:49:10
|
||||||
|
* @since 1.0.17
|
||||||
|
*/
|
||||||
|
public class MariaGtidLogEvent extends IgnorableLogEvent {
|
||||||
|
|
||||||
|
public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
|
||||||
|
super(header, buffer, descriptionEvent);
|
||||||
|
// do nothing , just ignore log event
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.nio.charset.Charset;
|
||||||
|
import java.util.BitSet;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
|
||||||
|
|
||||||
|
public class BaseLogFetcherTest {
|
||||||
|
|
||||||
|
protected String binlogFileName = "mysql-bin.000001";
|
||||||
|
protected Charset charset = Charset.forName("utf-8");
|
||||||
|
|
||||||
|
protected void parseQueryEvent(QueryLogEvent event) {
|
||||||
|
System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
|
||||||
|
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
|
||||||
|
event.getCatalog()));
|
||||||
|
System.out.println("sql : " + event.getQuery());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
|
||||||
|
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
|
||||||
|
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
|
||||||
|
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
|
||||||
|
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
|
||||||
|
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
|
||||||
|
System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void parseXidEvent(XidLogEvent event) throws Exception {
|
||||||
|
System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
|
||||||
|
event.getHeader().getLogPos() - event.getHeader().getEventLen()));
|
||||||
|
System.out.println("xid : " + event.getXid());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
protected void parseRowsEvent(RowsLogEvent event) {
|
||||||
|
try {
|
||||||
|
System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
|
||||||
|
event.getHeader().getLogPos() - event.getHeader().getEventLen(),
|
||||||
|
event.getTable().getDbName(), event.getTable().getTableName()));
|
||||||
|
RowsLogBuffer buffer = event.getRowsBuf(charset.name());
|
||||||
|
BitSet columns = event.getColumns();
|
||||||
|
BitSet changeColumns = event.getChangeColumns();
|
||||||
|
while (buffer.nextOneRow(columns)) {
|
||||||
|
// 处理row记录
|
||||||
|
int type = event.getHeader().getType();
|
||||||
|
if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
|
||||||
|
// insert的记录放在before字段中
|
||||||
|
parseOneRow(event, buffer, columns, true);
|
||||||
|
} else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
|
||||||
|
// delete的记录放在before字段中
|
||||||
|
parseOneRow(event, buffer, columns, false);
|
||||||
|
} else {
|
||||||
|
// update需要处理before/after
|
||||||
|
System.out.println("-------> before");
|
||||||
|
parseOneRow(event, buffer, columns, false);
|
||||||
|
if (!buffer.nextOneRow(changeColumns)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
System.out.println("-------> after");
|
||||||
|
parseOneRow(event, buffer, changeColumns, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException("parse row data failed.", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
|
||||||
|
throws UnsupportedEncodingException {
|
||||||
|
TableMapLogEvent map = event.getTable();
|
||||||
|
if (map == null) {
|
||||||
|
throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
|
||||||
|
}
|
||||||
|
|
||||||
|
final int columnCnt = map.getColumnCnt();
|
||||||
|
final ColumnInfo[] columnInfo = map.getColumnInfo();
|
||||||
|
|
||||||
|
for (int i = 0; i < columnCnt; i++) {
|
||||||
|
ColumnInfo info = columnInfo[i];
|
||||||
|
buffer.nextValue(info.type, info.meta);
|
||||||
|
|
||||||
|
if (buffer.isNull()) {
|
||||||
|
//
|
||||||
|
} else {
|
||||||
|
final Serializable value = buffer.getValue();
|
||||||
|
if (value instanceof byte[]) {
|
||||||
|
System.out.println(new String((byte[]) value));
|
||||||
|
} else {
|
||||||
|
System.out.println(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.Statement;
|
||||||
|
|
||||||
|
import junit.framework.Assert;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
|
||||||
|
|
||||||
|
public class DirectLogFetcherTest extends BaseLogFetcherTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSimple() {
|
||||||
|
DirectLogFetcher fecther = new DirectLogFetcher();
|
||||||
|
try {
|
||||||
|
Class.forName("com.mysql.jdbc.Driver");
|
||||||
|
Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "xxxxx", "xxxxx");
|
||||||
|
Statement statement = connection.createStatement();
|
||||||
|
statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
|
||||||
|
statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
|
||||||
|
|
||||||
|
fecther.open(connection, "mysql-bin.000003", 4L, 2);
|
||||||
|
|
||||||
|
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
|
||||||
|
LogContext context = new LogContext();
|
||||||
|
while (fecther.fetch()) {
|
||||||
|
LogEvent event = null;
|
||||||
|
event = decoder.decode(fecther, context);
|
||||||
|
|
||||||
|
if (event == null) {
|
||||||
|
throw new RuntimeException("parse failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
int eventType = event.getHeader().getType();
|
||||||
|
switch (eventType) {
|
||||||
|
case LogEvent.ROTATE_EVENT:
|
||||||
|
binlogFileName = ((RotateLogEvent) event).getFilename();
|
||||||
|
break;
|
||||||
|
case LogEvent.WRITE_ROWS_EVENT_V1:
|
||||||
|
case LogEvent.WRITE_ROWS_EVENT:
|
||||||
|
parseRowsEvent((WriteRowsLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.UPDATE_ROWS_EVENT_V1:
|
||||||
|
case LogEvent.UPDATE_ROWS_EVENT:
|
||||||
|
parseRowsEvent((UpdateRowsLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.DELETE_ROWS_EVENT_V1:
|
||||||
|
case LogEvent.DELETE_ROWS_EVENT:
|
||||||
|
parseRowsEvent((DeleteRowsLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.QUERY_EVENT:
|
||||||
|
parseQueryEvent((QueryLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.ROWS_QUERY_LOG_EVENT:
|
||||||
|
parseRowsQueryEvent((RowsQueryLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.ANNOTATE_ROWS_EVENT:
|
||||||
|
parseAnnotateRowsEvent((AnnotateRowsEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.XID_EVENT:
|
||||||
|
parseXidEvent((XidLogEvent) event);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
Assert.fail(e.getMessage());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
fecther.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
Assert.fail(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
import junit.framework.Assert;
|
||||||
|
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
|
||||||
|
import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
|
||||||
|
|
||||||
|
public class FileLogFetcherTest extends BaseLogFetcherTest {
|
||||||
|
|
||||||
|
private String directory;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
|
||||||
|
File dummyFile = new File(url.getFile());
|
||||||
|
directory = new File(dummyFile.getParent() + "/binlog").getPath();
|
||||||
|
// directory = "/home/jianghang/tmp/binlog";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSimple() {
|
||||||
|
FileLogFetcher fetcher = new FileLogFetcher(1024 * 16);
|
||||||
|
try {
|
||||||
|
LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
|
||||||
|
LogContext context = new LogContext();
|
||||||
|
|
||||||
|
File current = new File(directory, "mysql-bin.000006");
|
||||||
|
fetcher.open(current);
|
||||||
|
context.setLogPosition(new LogPosition(current.getName()));
|
||||||
|
|
||||||
|
while (fetcher.fetch()) {
|
||||||
|
LogEvent event = null;
|
||||||
|
event = decoder.decode(fetcher, context);
|
||||||
|
if (event != null) {
|
||||||
|
int eventType = event.getHeader().getType();
|
||||||
|
switch (eventType) {
|
||||||
|
case LogEvent.ROTATE_EVENT:
|
||||||
|
binlogFileName = ((RotateLogEvent)
|
||||||
|
event).getFilename();
|
||||||
|
break;
|
||||||
|
case LogEvent.WRITE_ROWS_EVENT_V1:
|
||||||
|
case LogEvent.WRITE_ROWS_EVENT:
|
||||||
|
parseRowsEvent((WriteRowsLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.UPDATE_ROWS_EVENT_V1:
|
||||||
|
case LogEvent.UPDATE_ROWS_EVENT:
|
||||||
|
parseRowsEvent((UpdateRowsLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.DELETE_ROWS_EVENT_V1:
|
||||||
|
case LogEvent.DELETE_ROWS_EVENT:
|
||||||
|
parseRowsEvent((DeleteRowsLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.QUERY_EVENT:
|
||||||
|
parseQueryEvent((QueryLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.ROWS_QUERY_LOG_EVENT:
|
||||||
|
parseRowsQueryEvent((RowsQueryLogEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.ANNOTATE_ROWS_EVENT:
|
||||||
|
parseAnnotateRowsEvent((AnnotateRowsEvent) event);
|
||||||
|
break;
|
||||||
|
case LogEvent.XID_EVENT:
|
||||||
|
parseXidEvent((XidLogEvent) event);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
Assert.fail(e.getMessage());
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
fetcher.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
Assert.fail(e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,356 @@
|
|||||||
|
package com.taobao.tddl.dbsync.binlog;
|
||||||
|
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.math.BigInteger;
|
||||||
|
|
||||||
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
|
import com.taobao.tddl.dbsync.binlog.LogBuffer;
|
||||||
|
|
||||||
|
public class LogBufferTest extends TestCase
|
||||||
|
{
|
||||||
|
public static final int LOOP = 10000;
|
||||||
|
|
||||||
|
public void testSigned()
|
||||||
|
{
|
||||||
|
byte[] array = { 0, 0, 0, (byte) 0xff };
|
||||||
|
|
||||||
|
LogBuffer buffer = new LogBuffer(array, 0, array.length);
|
||||||
|
|
||||||
|
System.out.println(buffer.getInt32(0));
|
||||||
|
System.out.println(buffer.getUint32(0));
|
||||||
|
|
||||||
|
System.out.println(buffer.getInt24(1));
|
||||||
|
System.out.println(buffer.getUint24(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testBigInteger()
|
||||||
|
{
|
||||||
|
byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
|
||||||
|
(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
|
||||||
|
|
||||||
|
LogBuffer buffer = new LogBuffer(array, 0, array.length);
|
||||||
|
|
||||||
|
long tt1 = 0;
|
||||||
|
long l1 = 0;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t1 = System.nanoTime();
|
||||||
|
l1 = buffer.getLong64(0);
|
||||||
|
tt1 += System.nanoTime() - t1;
|
||||||
|
}
|
||||||
|
System.out.print(tt1 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(l1);
|
||||||
|
|
||||||
|
long tt2 = 0;
|
||||||
|
BigInteger l2 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t2 = System.nanoTime();
|
||||||
|
l2 = buffer.getUlong64(0);
|
||||||
|
tt2 += System.nanoTime() - t2;
|
||||||
|
}
|
||||||
|
System.out.print(tt2 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(l2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Reads big-endian integer from no more than 4 bytes */
|
||||||
|
private static int convertNBytesToInt(byte[] buffer, int offset, int length)
|
||||||
|
{
|
||||||
|
int ret = 0;
|
||||||
|
for (int i = offset; i < (offset + length); i++)
|
||||||
|
{
|
||||||
|
ret = (ret << 8) | (0xff & buffer[i]);
|
||||||
|
}
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int convert4BytesToInt(byte[] buffer, int offset)
|
||||||
|
{
|
||||||
|
int value;
|
||||||
|
value = (0xff & buffer[offset + 3]);
|
||||||
|
value += (0xff & buffer[offset + 2]) << 8;
|
||||||
|
value += (0xff & buffer[offset + 1]) << 16;
|
||||||
|
value += (0xff & buffer[offset]) << 24;
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static short convert1ByteToShort(byte[] buffer, int offset)
|
||||||
|
{
|
||||||
|
short value;
|
||||||
|
value = (short) buffer[offset + 0];
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static short convert2bytesToShort(byte[] buffer, int offset)
|
||||||
|
{
|
||||||
|
short value;
|
||||||
|
value = (short) (buffer[offset + 0] << 8);
|
||||||
|
value += (short) (buffer[offset + 1] & 0xff);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final BigDecimal extractDecimal(byte[] buffer, int precision,
|
||||||
|
int scale)
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Decimal representation in binlog seems to be as follows:
|
||||||
|
// 1 byte - 'precision'
|
||||||
|
// 1 byte - 'scale'
|
||||||
|
// remaining n bytes - integer such that value = n / (10^scale)
|
||||||
|
// Integer is represented as follows:
|
||||||
|
// 1st bit - sign such that set == +, unset == -
|
||||||
|
// every 4 bytes represent 9 digits in big-endian order, so that if
|
||||||
|
// you print the values of these quads as big-endian integers one after
|
||||||
|
// another, you get the whole number string representation in decimal.
|
||||||
|
// What remains is to put a sign and a decimal dot.
|
||||||
|
// 13 0a 80 00 00 05 1b 38 b0 60 00 means:
|
||||||
|
// 0x13 - precision = 19
|
||||||
|
// 0x0a - scale = 10
|
||||||
|
// 0x80 - positive
|
||||||
|
// 0x00000005 0x1b38b060 0x00
|
||||||
|
// 5 456700000 0
|
||||||
|
// 54567000000 / 10^{10} = 5.4567
|
||||||
|
//
|
||||||
|
// int_size below shows how long is integer part
|
||||||
|
//
|
||||||
|
// offset = offset + 2; // offset of the number part
|
||||||
|
//
|
||||||
|
int intg = precision - scale;
|
||||||
|
int intg0 = intg / LogBuffer.DIG_PER_INT32;
|
||||||
|
int frac0 = scale / LogBuffer.DIG_PER_INT32;
|
||||||
|
int intg0x = intg - intg0 * LogBuffer.DIG_PER_INT32;
|
||||||
|
int frac0x = scale - frac0 * LogBuffer.DIG_PER_INT32;
|
||||||
|
|
||||||
|
int offset = 0;
|
||||||
|
|
||||||
|
int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
|
||||||
|
|
||||||
|
// how many bytes are used to represent given amount of digits?
|
||||||
|
int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
|
||||||
|
+ LogBuffer.dig2bytes[intg0x];
|
||||||
|
int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
|
||||||
|
+ LogBuffer.dig2bytes[frac0x];
|
||||||
|
|
||||||
|
int bin_size = integerSize + decimalSize; // total bytes
|
||||||
|
byte[] d_copy = new byte[bin_size];
|
||||||
|
|
||||||
|
if (bin_size > buffer.length)
|
||||||
|
{
|
||||||
|
throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
|
||||||
|
+ bin_size + ", available bytes: " + buffer.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Invert first bit
|
||||||
|
d_copy[0] = buffer[0];
|
||||||
|
d_copy[0] ^= 0x80;
|
||||||
|
if (sign == -1)
|
||||||
|
{
|
||||||
|
// Invert every byte
|
||||||
|
d_copy[0] ^= 0xFF;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 1; i < bin_size; i++)
|
||||||
|
{
|
||||||
|
d_copy[i] = buffer[i];
|
||||||
|
if (sign == -1)
|
||||||
|
{
|
||||||
|
// Invert every byte
|
||||||
|
d_copy[i] ^= 0xFF;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Integer part
|
||||||
|
offset = LogBuffer.dig2bytes[intg0x];
|
||||||
|
|
||||||
|
BigDecimal intPart = new BigDecimal(0);
|
||||||
|
|
||||||
|
if (offset > 0)
|
||||||
|
intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
|
||||||
|
|
||||||
|
while (offset < integerSize)
|
||||||
|
{
|
||||||
|
intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
|
||||||
|
BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
|
||||||
|
offset += 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decimal part
|
||||||
|
BigDecimal fracPart = new BigDecimal(0);
|
||||||
|
int shift = 0;
|
||||||
|
for (int i = 0; i < frac0; i++)
|
||||||
|
{
|
||||||
|
shift += LogBuffer.DIG_PER_DEC1;
|
||||||
|
fracPart = fracPart.add(BigDecimal.valueOf(
|
||||||
|
convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
|
||||||
|
offset += 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (LogBuffer.dig2bytes[frac0x] > 0)
|
||||||
|
{
|
||||||
|
fracPart = fracPart.add(BigDecimal.valueOf(
|
||||||
|
convertNBytesToInt(d_copy, offset,
|
||||||
|
LogBuffer.dig2bytes[frac0x])).movePointLeft(
|
||||||
|
shift + frac0x));
|
||||||
|
}
|
||||||
|
|
||||||
|
return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
|
||||||
|
0x38, (byte) 0xb0, 0x60, 0x00 };
|
||||||
|
|
||||||
|
public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
|
||||||
|
(byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
|
||||||
|
(byte) 0xa0, (byte) 0xff };
|
||||||
|
|
||||||
|
public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
|
||||||
|
|
||||||
|
public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
|
||||||
|
|
||||||
|
public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
|
||||||
|
-108 };
|
||||||
|
|
||||||
|
public void testBigDecimal() throws InterruptedException
|
||||||
|
{
|
||||||
|
do
|
||||||
|
{
|
||||||
|
System.out.println("old extract decimal: ");
|
||||||
|
|
||||||
|
long tt1 = 0;
|
||||||
|
BigDecimal bd1 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t1 = System.nanoTime();
|
||||||
|
bd1 = extractDecimal(array2, 19, 10);
|
||||||
|
tt1 += System.nanoTime() - t1;
|
||||||
|
}
|
||||||
|
System.out.print(tt1 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd1);
|
||||||
|
|
||||||
|
long tt2 = 0;
|
||||||
|
BigDecimal bd2 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t2 = System.nanoTime();
|
||||||
|
bd2 = extractDecimal(array1, 19, 10);
|
||||||
|
tt2 += System.nanoTime() - t2;
|
||||||
|
}
|
||||||
|
System.out.print(tt2 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd2);
|
||||||
|
|
||||||
|
long tt3 = 0;
|
||||||
|
BigDecimal bd3 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t3 = System.nanoTime();
|
||||||
|
bd3 = extractDecimal(array3, 18, 6);
|
||||||
|
tt3 += System.nanoTime() - t3;
|
||||||
|
}
|
||||||
|
System.out.print(tt3 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd3);
|
||||||
|
|
||||||
|
long tt4 = 0;
|
||||||
|
BigDecimal bd4 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t4 = System.nanoTime();
|
||||||
|
bd4 = extractDecimal(array4, 18, 6);
|
||||||
|
tt4 += System.nanoTime() - t4;
|
||||||
|
}
|
||||||
|
System.out.print(tt4 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd4);
|
||||||
|
|
||||||
|
long tt5 = 0;
|
||||||
|
BigDecimal bd5 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t5 = System.nanoTime();
|
||||||
|
bd5 = extractDecimal(array5, 18, 6);
|
||||||
|
tt5 += System.nanoTime() - t5;
|
||||||
|
}
|
||||||
|
System.out.print(tt5 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd5);
|
||||||
|
}
|
||||||
|
while (false);
|
||||||
|
|
||||||
|
do
|
||||||
|
{
|
||||||
|
System.out.println("new extract decimal: ");
|
||||||
|
|
||||||
|
LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
|
||||||
|
LogBuffer buffer2 = new LogBuffer(array1, 0, array1.length);
|
||||||
|
LogBuffer buffer3 = new LogBuffer(array3, 0, array3.length);
|
||||||
|
LogBuffer buffer4 = new LogBuffer(array4, 0, array4.length);
|
||||||
|
LogBuffer buffer5 = new LogBuffer(array5, 0, array5.length);
|
||||||
|
|
||||||
|
long tt1 = 0;
|
||||||
|
BigDecimal bd1 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t1 = System.nanoTime();
|
||||||
|
bd1 = buffer1.getDecimal(0, 19, 10);
|
||||||
|
tt1 += System.nanoTime() - t1;
|
||||||
|
}
|
||||||
|
System.out.print(tt1 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd1);
|
||||||
|
|
||||||
|
long tt2 = 0;
|
||||||
|
BigDecimal bd2 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t2 = System.nanoTime();
|
||||||
|
bd2 = buffer2.getDecimal(0, 19, 10);
|
||||||
|
tt2 += System.nanoTime() - t2;
|
||||||
|
}
|
||||||
|
System.out.print(tt2 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd2);
|
||||||
|
|
||||||
|
long tt3 = 0;
|
||||||
|
BigDecimal bd3 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t3 = System.nanoTime();
|
||||||
|
bd3 = buffer3.getDecimal(0, 18, 6);
|
||||||
|
tt3 += System.nanoTime() - t3;
|
||||||
|
}
|
||||||
|
System.out.print(tt3 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd3);
|
||||||
|
|
||||||
|
long tt4 = 0;
|
||||||
|
BigDecimal bd4 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t4 = System.nanoTime();
|
||||||
|
bd4 = buffer4.getDecimal(0, 18, 6);
|
||||||
|
tt4 += System.nanoTime() - t4;
|
||||||
|
}
|
||||||
|
System.out.print(tt4 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd4);
|
||||||
|
|
||||||
|
long tt5 = 0;
|
||||||
|
BigDecimal bd5 = null;
|
||||||
|
for (int i = 0; i < LOOP; i++)
|
||||||
|
{
|
||||||
|
final long t5 = System.nanoTime();
|
||||||
|
bd5 = buffer5.getDecimal(0, 18, 6);
|
||||||
|
tt5 += System.nanoTime() - t5;
|
||||||
|
}
|
||||||
|
System.out.print(tt5 / LOOP);
|
||||||
|
System.out.print("ns >> ");
|
||||||
|
System.out.println(bd5);
|
||||||
|
}
|
||||||
|
while (false);
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
@@ -0,0 +1 @@
|
|||||||
|
本文件仅仅为定位绝对路径使用
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal</artifactId>
|
||||||
|
<version>1.0.19-SNAPSHOT</version>
|
||||||
|
<relativePath>../pom.xml</relativePath>
|
||||||
|
</parent>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal.deployer</artifactId>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
<name>canal deployer module for otter ${project.version}</name>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba.otter</groupId>
|
||||||
|
<artifactId>canal.server</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 -->
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-jar-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<archive>
|
||||||
|
<addMavenDescriptor>true</addMavenDescriptor>
|
||||||
|
</archive>
|
||||||
|
<excludes>
|
||||||
|
<exclude>**/logback.xml</exclude>
|
||||||
|
<exclude>**/canal.properties</exclude>
|
||||||
|
<exclude>**/spring/**</exclude>
|
||||||
|
<exclude>**/example/**</exclude>
|
||||||
|
</excludes>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<!-- 这是最新版本，推荐使用这个版本 -->
|
||||||
|
<version>2.2.1</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>assemble</id>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<phase>package</phase>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<appendAssemblyId>false</appendAssemblyId>
|
||||||
|
<attach>false</attach>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>dev</id>
|
||||||
|
<activation>
|
||||||
|
<activeByDefault>true</activeByDefault>
|
||||||
|
<property>
|
||||||
|
<name>env</name>
|
||||||
|
<value>!release</value>
|
||||||
|
</property>
|
||||||
|
</activation>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
|
||||||
|
<descriptors>
|
||||||
|
<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
|
||||||
|
</descriptors>
|
||||||
|
<finalName>canal</finalName>
|
||||||
|
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
</profile>
|
||||||
|
|
||||||
|
<profile>
|
||||||
|
<id>release</id>
|
||||||
|
<activation>
|
||||||
|
<property>
|
||||||
|
<name>env</name>
|
||||||
|
<value>release</value>
|
||||||
|
</property>
|
||||||
|
</activation>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<!-- 发布模式使用的maven assembly插件描述文件 -->
|
||||||
|
<descriptors>
|
||||||
|
<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
|
||||||
|
</descriptors>
|
||||||
|
<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
|
||||||
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||||
|
<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
|
||||||
|
<outputDirectory>${project.parent.build.directory}</outputDirectory>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
|
</project>
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id>dist</id>
|
||||||
|
<formats>
|
||||||
|
<format>dir</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>.</directory>
|
||||||
|
<outputDirectory>/</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>README*</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>./src/main/bin</directory>
|
||||||
|
<outputDirectory>bin</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
<fileMode>0755</fileMode>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>./src/main/conf</directory>
|
||||||
|
<outputDirectory>/conf</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>./src/main/resources</directory>
|
||||||
|
<outputDirectory>/conf</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>target</directory>
|
||||||
|
<outputDirectory>logs</outputDirectory>
|
||||||
|
<excludes>
|
||||||
|
<exclude>**/*</exclude>
|
||||||
|
</excludes>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<outputDirectory>lib</outputDirectory>
|
||||||
|
<excludes>
|
||||||
|
<exclude>junit:junit</exclude>
|
||||||
|
</excludes>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
</assembly>
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id>dist</id>
|
||||||
|
<formats>
|
||||||
|
<format>tar.gz</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>.</directory>
|
||||||
|
<outputDirectory>/</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>README*</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>./src/main/bin</directory>
|
||||||
|
<outputDirectory>bin</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
<fileMode>0755</fileMode>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>./src/main/conf</directory>
|
||||||
|
<outputDirectory>/conf</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>./src/main/resources</directory>
|
||||||
|
<outputDirectory>/conf</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>**/*</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>target</directory>
|
||||||
|
<outputDirectory>logs</outputDirectory>
|
||||||
|
<excludes>
|
||||||
|
<exclude>**/*</exclude>
|
||||||
|
</excludes>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<outputDirectory>lib</outputDirectory>
|
||||||
|
<excludes>
|
||||||
|
<exclude>junit:junit</exclude>
|
||||||
|
</excludes>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
</assembly>
|
||||||
Executable
+25
@@ -0,0 +1,25 @@
|
|||||||
|
@echo off
|
||||||
|
@if not "%ECHO%" == "" echo %ECHO%
|
||||||
|
@if "%OS%" == "Windows_NT" setlocal
|
||||||
|
|
||||||
|
set ENV_PATH=.\
|
||||||
|
if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
|
||||||
|
|
||||||
|
set conf_dir=%ENV_PATH%\..\conf
|
||||||
|
set canal_conf=%conf_dir%\canal.properties
|
||||||
|
set logback_configurationFile=%conf_dir%\logback.xml
|
||||||
|
|
||||||
|
set CLASSPATH=%conf_dir%
|
||||||
|
set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
|
||||||
|
|
||||||
|
set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m
|
||||||
|
set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
|
||||||
|
set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n
|
||||||
|
set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%"
|
||||||
|
|
||||||
|
set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
|
||||||
|
|
||||||
|
set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher
|
||||||
|
echo start cmd : %CMD_STR%
|
||||||
|
|
||||||
|
java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
current_path=`pwd`
|
||||||
|
case "`uname`" in
|
||||||
|
Linux)
|
||||||
|
bin_abs_path=$(readlink -f $(dirname $0))
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
bin_abs_path=`cd $(dirname $0); pwd`
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
base=${bin_abs_path}/..
|
||||||
|
canal_conf=$base/conf/canal.properties
|
||||||
|
logback_configurationFile=$base/conf/logback.xml
|
||||||
|
export LANG=en_US.UTF-8
|
||||||
|
export BASE=$base
|
||||||
|
|
||||||
|
if [ -f $base/bin/canal.pid ] ; then
|
||||||
|
echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d $base/logs/canal ] ; then
|
||||||
|
mkdir -p $base/logs/canal
|
||||||
|
fi
|
||||||
|
|
||||||
|
## set java path
|
||||||
|
if [ -z "$JAVA" ] ; then
|
||||||
|
JAVA=$(which java)
|
||||||
|
fi
|
||||||
|
|
||||||
|
ALIBABA_JAVA="/usr/alibaba/java/bin/java"
|
||||||
|
TAOBAO_JAVA="/opt/taobao/java/bin/java"
|
||||||
|
if [ -z "$JAVA" ]; then
|
||||||
|
if [ -f $ALIBABA_JAVA ] ; then
|
||||||
|
JAVA=$ALIBABA_JAVA
|
||||||
|
elif [ -f $TAOBAO_JAVA ] ; then
|
||||||
|
JAVA=$TAOBAO_JAVA
|
||||||
|
else
|
||||||
|
echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
case "$#"
|
||||||
|
in
|
||||||
|
0 )
|
||||||
|
;;
|
||||||
|
1 )
|
||||||
|
var=$*
|
||||||
|
if [ -f $var ] ; then
|
||||||
|
canal_conf=$var
|
||||||
|
else
|
||||||
|
echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
|
||||||
|
exit
|
||||||
|
fi;;
|
||||||
|
2 )
|
||||||
|
var=$1
|
||||||
|
if [ -f $var ] ; then
|
||||||
|
canal_conf=$var
|
||||||
|
else
|
||||||
|
if [ "$1" = "debug" ]; then
|
||||||
|
DEBUG_PORT=$2
|
||||||
|
DEBUG_SUSPEND="n"
|
||||||
|
JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
|
||||||
|
fi
|
||||||
|
fi;;
|
||||||
|
* )
|
||||||
|
echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
|
||||||
|
exit;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
str=`file -L $JAVA | grep 64-bit`
|
||||||
|
if [ -n "$str" ]; then
|
||||||
|
JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError"
|
||||||
|
else
|
||||||
|
JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m "
|
||||||
|
fi
|
||||||
|
|
||||||
|
JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
|
||||||
|
CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
|
||||||
|
|
||||||
|
if [ -e $canal_conf -a -e $logback_configurationFile ]
|
||||||
|
then
|
||||||
|
|
||||||
|
for i in $base/lib/*;
|
||||||
|
do CLASSPATH=$i:"$CLASSPATH";
|
||||||
|
done
|
||||||
|
CLASSPATH="$base/conf:$CLASSPATH";
|
||||||
|
|
||||||
|
echo "cd to $bin_abs_path for workaround relative path"
|
||||||
|
cd $bin_abs_path
|
||||||
|
|
||||||
|
echo LOG CONFIGURATION : $logback_configurationFile
|
||||||
|
echo canal conf : $canal_conf
|
||||||
|
echo CLASSPATH :$CLASSPATH
|
||||||
|
$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 &
|
||||||
|
echo $! > $base/bin/canal.pid
|
||||||
|
|
||||||
|
echo "cd to $current_path for continue"
|
||||||
|
cd $current_path
|
||||||
|
else
|
||||||
|
echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!"
|
||||||
|
fi
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
cygwin=false;
|
||||||
|
case "`uname`" in
|
||||||
|
CYGWIN*)
|
||||||
|
cygwin=true
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
get_pid() {
|
||||||
|
STR=$1
|
||||||
|
PID=$2
|
||||||
|
if $cygwin; then
|
||||||
|
JAVA_CMD="$JAVA_HOME\bin\java"
|
||||||
|
JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
|
||||||
|
JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
|
||||||
|
else
|
||||||
|
if [ ! -z "$PID" ]; then
|
||||||
|
JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
|
||||||
|
else
|
||||||
|
JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
echo $JAVA_PID;
|
||||||
|
}
|
||||||
|
|
||||||
|
base=`dirname $0`/..
|
||||||
|
pidfile=$base/bin/canal.pid
|
||||||
|
if [ ! -f "$pidfile" ];then
|
||||||
|
echo "canal is not running. exists"
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
|
||||||
|
pid=`cat $pidfile`
|
||||||
|
if [ "$pid" == "" ] ; then
|
||||||
|
pid=`get_pid "appName=otter-canal"`
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "`hostname`: stopping canal $pid ... "
|
||||||
|
kill $pid
|
||||||
|
|
||||||
|
LOOPS=0
|
||||||
|
while (true);
|
||||||
|
do
|
||||||
|
gpid=`get_pid "appName=otter-canal" "$pid"`
|
||||||
|
if [ "$gpid" == "" ] ; then
|
||||||
|
echo "Oook! cost:$LOOPS"
|
||||||
|
`rm $pidfile`
|
||||||
|
break;
|
||||||
|
fi
|
||||||
|
let LOOPS=LOOPS+1
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package com.alibaba.otter.canal.deployer;
|
||||||
|
|
||||||
|
import java.text.MessageFormat;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 启动常用变量
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-11-8 下午03:15:55
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class CanalConstants {
|
||||||
|
|
||||||
|
public static final String MDC_DESTINATION = "destination";
|
||||||
|
public static final String ROOT = "canal";
|
||||||
|
public static final String CANAL_ID = ROOT + "." + "id";
|
||||||
|
public static final String CANAL_IP = ROOT + "." + "ip";
|
||||||
|
public static final String CANAL_PORT = ROOT + "." + "port";
|
||||||
|
public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers";
|
||||||
|
|
||||||
|
public static final String CANAL_DESTINATIONS = ROOT + "." + "destinations";
|
||||||
|
public static final String CANAL_AUTO_SCAN = ROOT + "." + "auto.scan";
|
||||||
|
public static final String CANAL_AUTO_SCAN_INTERVAL = ROOT + "." + "auto.scan.interval";
|
||||||
|
public static final String CANAL_CONF_DIR = ROOT + "." + "conf.dir";
|
||||||
|
|
||||||
|
public static final String CANAL_DESTINATION_SPLIT = ",";
|
||||||
|
public static final String GLOBAL_NAME = "global";
|
||||||
|
|
||||||
|
public static final String INSTANCE_MODE_TEMPLATE = ROOT + "." + "instance.{0}.mode";
|
||||||
|
public static final String INSTANCE_LAZY_TEMPLATE = ROOT + "." + "instance.{0}.lazy";
|
||||||
|
public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address";
|
||||||
|
public static final String INSTANCE_SPRING_XML_TEMPLATE = ROOT + "." + "instance.{0}.spring.xml";
|
||||||
|
|
||||||
|
public static final String CANAL_DESTINATION_PROPERTY = ROOT + ".instance.destination";
|
||||||
|
|
||||||
|
public static String getInstanceModeKey(String destination) {
|
||||||
|
return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getInstanceManagerAddressKey(String destination) {
|
||||||
|
return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getInstancSpringXmlKey(String destination) {
|
||||||
|
return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getInstancLazyKey(String destination) {
|
||||||
|
return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,452 @@
|
|||||||
|
package com.alibaba.otter.canal.deployer;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.I0Itec.zkclient.IZkStateListener;
|
||||||
|
import org.I0Itec.zkclient.exception.ZkNoNodeException;
|
||||||
|
import org.apache.commons.lang.BooleanUtils;
|
||||||
|
import org.apache.commons.lang.StringUtils;
|
||||||
|
import org.apache.zookeeper.Watcher.Event.KeeperState;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.slf4j.MDC;
|
||||||
|
import org.springframework.beans.factory.BeanFactory;
|
||||||
|
import org.springframework.context.ApplicationContext;
|
||||||
|
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||||
|
|
||||||
|
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
|
||||||
|
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
|
||||||
|
import com.alibaba.otter.canal.deployer.InstanceConfig.InstanceMode;
|
||||||
|
import com.alibaba.otter.canal.deployer.monitor.InstanceAction;
|
||||||
|
import com.alibaba.otter.canal.deployer.monitor.InstanceConfigMonitor;
|
||||||
|
import com.alibaba.otter.canal.deployer.monitor.ManagerInstanceConfigMonitor;
|
||||||
|
import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor;
|
||||||
|
import com.alibaba.otter.canal.instance.core.CanalInstance;
|
||||||
|
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
|
||||||
|
import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
|
||||||
|
import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
|
||||||
|
import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
|
||||||
|
import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
|
||||||
|
import com.alibaba.otter.canal.server.exception.CanalServerException;
|
||||||
|
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
|
||||||
|
import com.google.common.base.Function;
|
||||||
|
import com.google.common.collect.MapMaker;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* canal调度控制器
|
||||||
|
*
|
||||||
|
* @author jianghang 2012-11-8 下午12:03:11
|
||||||
|
* @version 1.0.0
|
||||||
|
*/
|
||||||
|
public class CanalController {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(CanalController.class);
|
||||||
|
private Long cid;
|
||||||
|
private String ip;
|
||||||
|
private int port;
|
||||||
|
// 默认使用spring的方式载入
|
||||||
|
private Map<String, InstanceConfig> instanceConfigs;
|
||||||
|
private InstanceConfig globalInstanceConfig;
|
||||||
|
private Map<String, CanalConfigClient> managerClients;
|
||||||
|
// 监听instance config的变化
|
||||||
|
private boolean autoScan = true;
|
||||||
|
private InstanceAction defaultAction;
|
||||||
|
private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
|
||||||
|
private CanalServerWithEmbeded embededCanalServer;
|
||||||
|
private CanalServerWithNetty canalServer;
|
||||||
|
|
||||||
|
private CanalInstanceGenerator instanceGenerator;
|
||||||
|
private ZkClientx zkclientx;
|
||||||
|
|
||||||
|
public CanalController(){
|
||||||
|
this(System.getProperties());
|
||||||
|
}
|
||||||
|
|
||||||
|
public CanalController(final Properties properties){
|
||||||
|
managerClients = new MapMaker().makeComputingMap(new Function<String, CanalConfigClient>() {
|
||||||
|
|
||||||
|
public CanalConfigClient apply(String managerAddress) {
|
||||||
|
return getManagerClient(managerAddress);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// 初始化全局参数设置
|
||||||
|
globalInstanceConfig = initGlobalConfig(properties);
|
||||||
|
instanceConfigs = new MapMaker().makeMap();
|
||||||
|
// 初始化instance config
|
||||||
|
initInstanceConfig(properties);
|
||||||
|
|
||||||
|
// 准备canal server
|
||||||
|
cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
|
||||||
|
ip = getProperty(properties, CanalConstants.CANAL_IP);
|
||||||
|
port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT));
|
||||||
|
embededCanalServer = new CanalServerWithEmbeded();
|
||||||
|
embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
|
||||||
|
canalServer = new CanalServerWithNetty(embededCanalServer);
|
||||||
|
canalServer.setIp(ip);
|
||||||
|
canalServer.setPort(port);
|
||||||
|
|
||||||
|
// 处理下ip为空，默认使用hostIp暴露到zk中
|
||||||
|
if (StringUtils.isEmpty(ip)) {
|
||||||
|
ip = AddressUtils.getHostIp();
|
||||||
|
}
|
||||||
|
final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS);
|
||||||
|
if (StringUtils.isNotEmpty(zkServers)) {
|
||||||
|
zkclientx = ZkClientx.getZkClient(zkServers);
|
||||||
|
// 初始化系统目录
|
||||||
|
zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true);
|
||||||
|
zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
|
||||||
|
ServerRunningMonitors.setServerData(serverData);
|
||||||
|
ServerRunningMonitors.setRunningMonitors(new MapMaker().makeComputingMap(new Function<String, ServerRunningMonitor>() {
|
||||||
|
|
||||||
|
public ServerRunningMonitor apply(final String destination) {
|
||||||
|
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
|
||||||
|
runningMonitor.setDestination(destination);
|
||||||
|
runningMonitor.setListener(new ServerRunningListener() {
|
||||||
|
|
||||||
|
public void processActiveEnter() {
|
||||||
|
try {
|
||||||
|
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
||||||
|
embededCanalServer.start(destination);
|
||||||
|
} finally {
|
||||||
|
MDC.remove(CanalConstants.MDC_DESTINATION);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processActiveExit() {
|
||||||
|
try {
|
||||||
|
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
||||||
|
embededCanalServer.stop(destination);
|
||||||
|
} finally {
|
||||||
|
MDC.remove(CanalConstants.MDC_DESTINATION);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processStart() {
|
||||||
|
try {
|
||||||
|
if (zkclientx != null) {
|
||||||
|
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
|
||||||
|
+ port);
|
||||||
|
initCid(path);
|
||||||
|
zkclientx.subscribeStateChanges(new IZkStateListener() {
|
||||||
|
|
||||||
|
public void handleStateChanged(KeeperState state) throws Exception {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleNewSession() throws Exception {
|
||||||
|
initCid(path);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
MDC.remove(CanalConstants.MDC_DESTINATION);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void processStop() {
|
||||||
|
try {
|
||||||
|
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
||||||
|
if (zkclientx != null) {
|
||||||
|
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
|
||||||
|
+ port);
|
||||||
|
releaseCid(path);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
MDC.remove(CanalConstants.MDC_DESTINATION);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
if (zkclientx != null) {
|
||||||
|
runningMonitor.setZkClient(zkclientx);
|
||||||
|
}
|
||||||
|
return runningMonitor;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 初始化monitor机制
|
||||||
|
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
|
||||||
|
if (autoScan) {
|
||||||
|
defaultAction = new InstanceAction() {
|
||||||
|
|
||||||
|
public void start(String destination) {
|
||||||
|
InstanceConfig config = instanceConfigs.get(destination);
|
||||||
|
if (config == null) {
|
||||||
|
config = new InstanceConfig(globalInstanceConfig);
|
||||||
|
instanceConfigs.put(destination, config);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
|
||||||
|
// HA机制启动
|
||||||
|
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
|
||||||
|
if (!runningMonitor.isStart()) {
|
||||||
|
runningMonitor.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop(String destination) {
|
||||||
|
// 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息
|
||||||
|
InstanceConfig config = instanceConfigs.remove(destination);
|
||||||
|
if (config != null) {
|
||||||
|
embededCanalServer.stop(destination);
|
||||||
|
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
|
||||||
|
if (runningMonitor.isStart()) {
|
||||||
|
runningMonitor.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void reload(String destination) {
|
||||||
|
// 目前任何配置变化，直接重启，简单处理
|
||||||
|
stop(destination);
|
||||||
|
start(destination);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
instanceConfigMonitors = new MapMaker().makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
|
||||||
|
|
||||||
|
public InstanceConfigMonitor apply(InstanceMode mode) {
|
||||||
|
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
|
||||||
|
|
||||||
|
if (mode.isSpring()) {
|
||||||
|
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
|
||||||
|
monitor.setScanIntervalInSecond(scanInterval);
|
||||||
|
monitor.setDefaultAction(defaultAction);
|
||||||
|
// 设置conf目录，默认是user.dir + conf目录组成
|
||||||
|
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
|
||||||
|
if (StringUtils.isEmpty(rootDir)) {
|
||||||
|
rootDir = "../conf";
|
||||||
|
}
|
||||||
|
monitor.setRootConf(rootDir);
|
||||||
|
return monitor;
|
||||||
|
} else if (mode.isManager()) {
|
||||||
|
return new ManagerInstanceConfigMonitor();
|
||||||
|
} else {
|
||||||
|
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private InstanceConfig initGlobalConfig(Properties properties) {
|
||||||
|
InstanceConfig globalConfig = new InstanceConfig();
|
||||||
|
String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME));
|
||||||
|
if (StringUtils.isNotEmpty(modeStr)) {
|
||||||
|
globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
|
||||||
|
}
|
||||||
|
|
||||||
|
String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME));
|
||||||
|
if (StringUtils.isNotEmpty(lazyStr)) {
|
||||||
|
globalConfig.setLazy(Boolean.valueOf(lazyStr));
|
||||||
|
}
|
||||||
|
|
||||||
|
String managerAddress = getProperty(properties,
|
||||||
|
CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME));
|
||||||
|
if (StringUtils.isNotEmpty(managerAddress)) {
|
||||||
|
globalConfig.setManagerAddress(managerAddress);
|
||||||
|
}
|
||||||
|
|
||||||
|
String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME));
|
||||||
|
if (StringUtils.isNotEmpty(springXml)) {
|
||||||
|
globalConfig.setSpringXml(springXml);
|
||||||
|
}
|
||||||
|
|
||||||
|
instanceGenerator = new CanalInstanceGenerator() {
|
||||||
|
|
||||||
|
public CanalInstance generate(String destination) {
|
||||||
|
InstanceConfig config = instanceConfigs.get(destination);
|
||||||
|
if (config == null) {
|
||||||
|
throw new CanalServerException("can't find destination:{}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.getMode().isManager()) {
|
||||||
|
ManagerCanalInstanceGenerator instanceGenerator = new ManagerCanalInstanceGenerator();
|
||||||
|
instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
|
||||||
|
return instanceGenerator.generate(destination);
|
||||||
|
} else if (config.getMode().isSpring()) {
|
||||||
|
SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
|
||||||
|
synchronized (this) {
|
||||||
|
try {
|
||||||
|
// 设置当前正在加载的通道，加载spring查找文件时会用到该变量
|
||||||
|
System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination);
|
||||||
|
instanceGenerator.setBeanFactory(getBeanFactory(config.getSpringXml()));
|
||||||
|
return instanceGenerator.generate(destination);
|
||||||
|
} finally {
|
||||||
|
System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new UnsupportedOperationException("unknow mode :" + config.getMode());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
return globalConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
private CanalConfigClient getManagerClient(String managerAddress) {
|
||||||
|
return new CanalConfigClient();
|
||||||
|
}
|
||||||
|
|
||||||
|
private BeanFactory getBeanFactory(String springXml) {
|
||||||
|
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
|
||||||
|
return applicationContext;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initInstanceConfig(Properties properties) {
|
||||||
|
String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
|
||||||
|
String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT);
|
||||||
|
|
||||||
|
for (String destination : destinations) {
|
||||||
|
InstanceConfig config = parseInstanceConfig(properties, destination);
|
||||||
|
InstanceConfig oldConfig = instanceConfigs.put(destination, config);
|
||||||
|
|
||||||
|
if (oldConfig != null) {
|
||||||
|
logger.warn("destination:{} old config:{} has replace by new config:{}", new Object[] { destination,
|
||||||
|
oldConfig, config });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private InstanceConfig parseInstanceConfig(Properties properties, String destination) {
|
||||||
|
InstanceConfig config = new InstanceConfig(globalInstanceConfig);
|
||||||
|
String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination));
|
||||||
|
if (!StringUtils.isEmpty(modeStr)) {
|
||||||
|
config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
|
||||||
|
}
|
||||||
|
|
||||||
|
String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination));
|
||||||
|
if (!StringUtils.isEmpty(lazyStr)) {
|
||||||
|
config.setLazy(Boolean.valueOf(lazyStr));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config.getMode().isManager()) {
|
||||||
|
String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination));
|
||||||
|
if (StringUtils.isNotEmpty(managerAddress)) {
|
||||||
|
config.setManagerAddress(managerAddress);
|
||||||
|
}
|
||||||
|
} else if (config.getMode().isSpring()) {
|
||||||
|
String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination));
|
||||||
|
if (StringUtils.isNotEmpty(springXml)) {
|
||||||
|
config.setSpringXml(springXml);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getProperty(Properties properties, String key) {
|
||||||
|
return StringUtils.trim(properties.getProperty(StringUtils.trim(key)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void start() throws Throwable {
|
||||||
|
logger.info("## start the canal server[{}:{}]", ip, port);
|
||||||
|
// 创建整个canal的工作节点
|
||||||
|
final String path = ZookeeperPathUtils.getCanalClusterNode(ip + ":" + port);
|
||||||
|
initCid(path);
|
||||||
|
if (zkclientx != null) {
|
||||||
|
this.zkclientx.subscribeStateChanges(new IZkStateListener() {
|
||||||
|
|
||||||
|
public void handleStateChanged(KeeperState state) throws Exception {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public void handleNewSession() throws Exception {
|
||||||
|
initCid(path);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// 优先启动embeded服务
|
||||||
|
embededCanalServer.start();
|
||||||
|
// 尝试启动一下非lazy状态的通道
|
||||||
|
for (Map.Entry<String, InstanceConfig> entry : instanceConfigs.entrySet()) {
|
||||||
|
final String destination = entry.getKey();
|
||||||
|
InstanceConfig config = entry.getValue();
|
||||||
|
// 创建destination的工作节点
|
||||||
|
if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
|
||||||
|
// HA机制启动
|
||||||
|
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
|
||||||
|
if (!runningMonitor.isStart()) {
|
||||||
|
runningMonitor.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (autoScan) {
|
||||||
|
instanceConfigMonitors.get(config.getMode()).regeister(destination, defaultAction);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (autoScan) {
|
||||||
|
instanceConfigMonitors.get(globalInstanceConfig.getMode()).start();
|
||||||
|
for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
|
||||||
|
if (!monitor.isStart()) {
|
||||||
|
monitor.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 启动网络接口
|
||||||
|
canalServer.start();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void stop() throws Throwable {
|
||||||
|
canalServer.stop();
|
||||||
|
|
||||||
|
if (autoScan) {
|
||||||
|
for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
|
||||||
|
if (monitor.isStart()) {
|
||||||
|
monitor.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) {
|
||||||
|
if (runningMonitor.isStart()) {
|
||||||
|
runningMonitor.stop();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 释放canal的工作节点
|
||||||
|
releaseCid(ZookeeperPathUtils.getCanalClusterNode(ip + ":" + port));
|
||||||
|
logger.info("## stop the canal server[{}:{}]", ip, port);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initCid(String path) {
|
||||||
|
// logger.info("## init the canalId = {}", cid);
|
||||||
|
// 初始化系统目录
|
||||||
|
if (zkclientx != null) {
|
||||||
|
try {
|
||||||
|
zkclientx.createEphemeral(path);
|
||||||
|
} catch (ZkNoNodeException e) {
|
||||||
|
// 如果父目录不存在，则创建
|
||||||
|
String parentDir = path.substring(0, path.lastIndexOf('/'));
|
||||||
|
zkclientx.createPersistent(parentDir, true);
|
||||||
|
zkclientx.createEphemeral(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void releaseCid(String path) {
|
||||||
|
// logger.info("## release the canalId = {}", cid);
|
||||||
|
// 初始化系统目录
|
||||||
|
if (zkclientx != null) {
|
||||||
|
zkclientx.delete(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user