wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #16 from alibaba/master

Browse Source 
merge
This commit is contained in:
rewerma
2018-10-29 20:14:36 +08:00
committed by GitHub
parent b131e8cfa4 be5f1546ca
commit 7efd067bd5
30 changed files with 80 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client-adapter/README.md
+1 -1
View File
@@ -41,7 +41,7 @@ canal.conf:
flatMessage: true # 扁平message开关, 是否以json字符串形式投递数据, 仅在kafka/rocketMQ模式下有效
canalInstances: # canal实例组, 如果是tcp模式可配置此项
- instance: example # 对应canal destination
adapterGroups: # 对应适配器分组, 分组间的适配器并行运行
groups: # 对应适配器分组, 分组间的适配器并行运行
- outAdapters: # 适配器列表, 分组内的适配串行运行
- name: logger # 适配器SPI名
- name: hbase
client-adapter/common/pom.xml
+3 -5
View File
@@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
@@ -16,7 +14,7 @@
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${canal_version}</version>
<version>1.1.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
client-adapter/hbase/pom.xml
+2 -4
View File
@@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
client-adapter/launcher/pom.xml
+7 -9
View File
@@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
@@ -34,7 +32,7 @@
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client</artifactId>
<version>${canal_version}</version>
<version>1.1.2-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
@@ -134,14 +132,14 @@
</goals>
<configuration>
<tasks>
<copy todir="${project.basedir}/target/config" overwrite="true" >
<copy todir="${project.basedir}/target/config" overwrite="true">
<fileset dir="${project.basedir}/src/main/resources" erroronmissingdir="true">
<include name="*.yml"/>
<include name="*.yml" />
</fileset>
</copy>
<copy todir="${project.basedir}/target/config/hbase" overwrite="true" >
<copy todir="${project.basedir}/target/config/hbase" overwrite="true">
<fileset dir="${project.basedir}/../hbase/src/main/resources/hbase" erroronmissingdir="true">
<include name="*.yml"/>
<include name="*.yml" />
</fileset>
</copy>
</tasks>
client-adapter/logger/pom.xml
+2 -4
View File
@@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>canal.client-adapter</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
client-adapter/pom.xml
+2 -4
View File
@@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.client-adapter</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>canal client adapter module for otter ${project.version}</name>
client/pom.xml
+2 -3
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
@@ -180,8 +180,7 @@
</relocation>
</relocations>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
</transformers>
</configuration>
</execution>
common/pom.xml
+5 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>canal.common</artifactId>
@@ -83,6 +83,10 @@
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java
+1 -1
View File
@@ -5,7 +5,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.AbstractQueuedSynchronizer;
/**
* 实现一个互斥实现，基于Cocurrent中的{@linkplain AbstractQueuedSynchronizer}实现了自己的sync <br/>
* 实现一个互斥实现，基于Cocurrent中的AbstractQueuedSynchronizer实现了自己的sync <br/>
* 应用场景：系统初始化/授权控制，没权限时阻塞等待。有权限时所有线程都可以快速通过
*
* <pre>
dbsync/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
deployer/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
deployer/src/main/resources/example/instance.properties
+3 -3
View File
@@ -31,12 +31,12 @@ canal.instance.tsdb.enable=true
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=cZozNf1mzW6EQLGO2q9u99619xbZLO0fbua3EX08r4BWNXb8lAt1aHrTEOBttd6UY8Vnuc0easlVXZDdLtt8BQ==
canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
canal.instance.dbPassword=canal
canal.instance.connectionCharset = UTF-8
canal.instance.defaultDatabaseName =test
# enable druid Decrypt database password
canal.instance.enableDruid=true
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
# table regex
canal.instance.filter.regex=.*\\..*
driver/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
example/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
filter/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
instance/core/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>canal.instance.core</artifactId>
instance/manager/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
instance/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
instance/spring/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
meta/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
parse/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>canal.parse</artifactId>
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
+4 -4
View File
@@ -35,12 +35,12 @@ import com.taobao.tddl.dbsync.binlog.LogEvent;
/**
* 基于向mysql server复制binlog实现
*
*
* <pre>
* 1. 自身不控制mysql主备切换，由ha机制来控制. 比如接入tddl/cobar/自身心跳包成功率
* 2. 切换机制
* </pre>
*
*
* @author jianghang 2012-6-21 下午04:06:32
* @version 1.0.0
*/
@@ -202,7 +202,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
/**
* 心跳信息
*
*
* @author jianghang 2012-7-6 下午02:50:15
* @version 1.0.0
*/
@@ -352,7 +352,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
return logPosition.getPostion();
}
if (StringUtils.isNotEmpty(masterPosition.getGtid())) {
if (masterPosition!=null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
return masterPosition;
}
}
parse/src/test/java/com/alibaba/otter/canal/parse/inbound/mysql/tsdb/FastsqlSchemaTest.java
+2
View File
@@ -19,6 +19,8 @@ public class FastsqlSchemaTest {
public void testSimple() throws FileNotFoundException, IOException {
SchemaRepository repository = new SchemaRepository(JdbcConstants.MYSQL);
String sql = "create table yushitai_test.card_record ( id bigint auto_increment, name varchar(32) DEFAULT NULL) auto_increment=256 "
+ "ALTER TABLE yushitai_test.card_record ADD COLUMN remark2 varchar(255) DEFAULT NULL , ALGORITHM=inplace,LOCK=NONE;"
+ "ALTER TABLE yushitai_test.card_record modify COLUMN name varchar(64) DEFAULT NULL , ALGORITHM=copy,LOCK=SHARED; "
+ "alter table yushitai_test.card_record add index index_name(name) ;"
+ "alter table yushitai_test.card_record add index index_name(name) ;"
+ "alter table yushitai_test.card_record add Constraint pk_id PRIMARY KEY (id);"
pom.xml
+25 -3
View File
@@ -4,7 +4,7 @@
<artifactId>canal</artifactId>
<packaging>pom</packaging>
<name>canal module for otter ${project.version}</name>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<url>https://github.com/alibaba/canal</url>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -246,7 +246,7 @@
<dependency>
<groupId>com.alibaba.fastsql</groupId>
<artifactId>fastsql</artifactId>
<version>2.0.0_preview_630</version>
<version>2.0.0_preview_644</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
@@ -297,7 +297,11 @@
<version>5.1.40</version>
<!--<scope>test</scope>-->
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>3.0.2</version>
</dependency>
</dependencies>
</dependencyManagement>
@@ -367,6 +371,24 @@
</configuration>
</plugin>
<!-- javadoc -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
<encoding>${file_encoding}</encoding>
<charset>${file_encoding}</charset>
<additionalparam>-Xdoclint:none</additionalparam>
</configuration>
</plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
prometheus/pom.xml
+2 -2
View File
@@ -3,13 +3,13 @@
<parent>
<artifactId>canal</artifactId>
<groupId>com.alibaba.otter</groupId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.prometheus</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<name>canal prometheus module for otter ${project.version}</name>
<dependencies>
<dependency>
protocol/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
server/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>canal.server</artifactId>
server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java
+4 -2
View File
@@ -63,7 +63,6 @@ public class CanalRocketMQProducer implements CanalMQProducer {
return mqs.get(partition);
}
}, null);
callback.commit();
} catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
logger.error("Send message error!", e);
callback.rollback();
@@ -122,7 +121,6 @@ public class CanalRocketMQProducer implements CanalMQProducer {
logger.error("send flat message to hashed partition error", e);
callback.rollback();
}
}
}
}
@@ -130,6 +128,10 @@ public class CanalRocketMQProducer implements CanalMQProducer {
}
}
callback.commit();
if (logger.isDebugEnabled()) {
logger.debug("send message to rocket topic: {}", destination.getTopic());
}
}
@Override
sink/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>
store/pom.xml
+1 -1
View File
@@ -3,7 +3,7 @@
<parent>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<version>1.1.1-SNAPSHOT</version>
<version>1.1.2-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.alibaba.otter</groupId>