wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

es配置加载

Browse Source
This commit is contained in:
mcy
2018-10-30 15:57:00 +08:00
parent e4e23c9ecd
commit e1191b665e
7 changed files with 105 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java
+5 -12
View File
@@ -28,24 +28,17 @@ import com.alibaba.otter.canal.client.adapter.support.SPI;
@SPI("es")
public class ESAdapter implements OuterAdapter {
private static Logger logger = LoggerFactory.getLogger(ESAdapter.class);
private static Logger logger = LoggerFactory.getLogger(ESAdapter.class);
private static volatile Map<String, ESSyncConfig> esSyncConfig = null; // 文件名对应配置
private TransportClient transportClient;
private TransportClient transportClient;
private ESSyncService esSyncService;
private ESSyncService esSyncService;
@Override
public void init(OuterAdapterConfig configuration) {
try {
if (esSyncConfig == null) {
synchronized (ESSyncConfig.class) {
if (esSyncConfig == null) {
esSyncConfig = ESSyncConfigLoader.load();
}
}
}
ESSyncConfigLoader.load();
Map<String, String> properties = configuration.getProperties();
Settings.Builder settingBuilder = Settings.builder();
properties.forEach(settingBuilder::put);
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java
+9
View File
@@ -53,6 +53,8 @@ public class ESSyncConfig {
private int commitBatch = 1000;
private String etlCondition;
private SchemaItem schemaItem; // sql解析结果模型
public String get_index() {
return _index;
}
@@ -133,5 +135,12 @@ public class ESSyncConfig {
this.etlCondition = etlCondition;
}
public SchemaItem getSchemaItem() {
return schemaItem;
}
public void setSchemaItem(SchemaItem schemaItem) {
this.schemaItem = schemaItem;
}
}
}
client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java
+43 -14
View File
@@ -5,27 +5,38 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
public class ESSyncConfigLoader {
private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class);
private static Logger logger = LoggerFactory
.getLogger(ESSyncConfigLoader.class);
private static final String BASE_PATH = "es";
private static final String BASE_PATH = "es";
public static Map<String, ESSyncConfig> load() {
private static volatile Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>(); // 文件名对应配置
private static volatile Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = new LinkedHashMap<>(); // schema-table对应配置
public static Map<String, ESSyncConfig> getEsSyncConfig() {
return esSyncConfig;
}
public static Map<String, List<ESSyncConfig>> getDbTableEsSyncConfig() {
return dbTableEsSyncConfig;
}
public static synchronized void load() {
logger.info("## Start loading mapping config ... ");
Map<String, ESSyncConfig> result = new LinkedHashMap<>();
Collection<String> configs = AdapterConfigs.get("es");
for (String c : configs) {
if (c == null) {
@@ -47,17 +58,35 @@ public class ESSyncConfigLoader {
try {
config.validate();
SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
config.getEsMapping().setSchemaItem(schemaItem);
DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
if (dataSource == null || dataSource.getUrl() == null) {
throw new RuntimeException("No data source found: " + config.getDataSourceKey());
}
Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
Matcher matcher = pattern.matcher(dataSource.getUrl());
if (!matcher.find()){
throw new RuntimeException("No found the schema of jdbc-url: " + config.getDataSourceKey());
}
String schema = matcher.group(2);
schemaItem.getAliasTableItems().values().forEach(tableItem -> {
List<ESSyncConfig> esSyncConfigs = dbTableEsSyncConfig
.computeIfAbsent(schema + "-" + tableItem.getTableName(), k -> new ArrayList<>());
esSyncConfigs.add(config);
});
} catch (Exception e) {
throw new RuntimeException("ERROR Config: " + c + " " + e.getMessage(), e);
throw new RuntimeException("ERROR Config: " + c, e);
}
result.put(c, config);
esSyncConfig.put(c, config);
}
logger.info("## Mapping config loaded");
return result;
}
public static String readConfigContent(String config) {
private static String readConfigContent(String config) {
InputStream in = null;
try {
// 先取本地文件，再取类路径
@@ -68,7 +97,7 @@ public class ESSyncConfigLoader {
in = ESSyncConfigLoader.class.getClassLoader().getResourceAsStream(config);
}
if (in == null) {
throw new RuntimeException("Config file not found.");
throw new RuntimeException("Config file: " + config + " not found.");
}
byte[] bytes = new byte[in.available()];
client-adapter/elasticsearch/src/main/resources/es/myetst_user.yml
+1 -1
View File
@@ -4,7 +4,7 @@ esMapping:
_type: _doc
_id: id
# pk: id
sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user.a
sql: "select a.id, concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a
left join role b on a.role_id=b.id
left join (select user_id, group_concat(label,',') as labels from user_label
group by user_id) c on c.user_id=a.id"
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java
+16 -6
View File
@@ -1,24 +1,31 @@
package com.alibaba.otter.canal.client.adapter.es.test;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Map;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
import com.alibaba.otter.canal.client.adapter.support.AdapterConfigs;
import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
public class ConfigLoadTest {
@Before
public void before() {
public void before() throws SQLException {
AdapterConfigs.put("es", "myetst_user.yml");
// 加载数据源连接池
DatasourceConfig.DATA_SOURCES.put("defaultDS", DataSourceConstant.dataSource);
}
@Test
public void testLoad() {
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load();
ESSyncConfigLoader.load();
Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.getEsSyncConfig();
ESSyncConfig config = configMap.get("myetst_user.yml");
Assert.assertNotNull(config);
Assert.assertEquals("defaultDS", config.getDataSourceKey());
@@ -27,5 +34,8 @@ public class ConfigLoadTest {
Assert.assertEquals("_doc", esMapping.get_type());
Assert.assertEquals("id", esMapping.get_id());
Assert.assertNotNull(esMapping.getSql());
Map<String, List<ESSyncConfig>> dbTableEsSyncConfig = ESSyncConfigLoader.getDbTableEsSyncConfig();
Assert.assertFalse(dbTableEsSyncConfig.isEmpty());
}
}
client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/DataSourceConstant.java
+31
View File
@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.client.adapter.es.test;
import java.sql.SQLException;
import com.alibaba.druid.pool.DruidDataSource;
public class DataSourceConstant {
static DruidDataSource dataSource;
static {
dataSource = new DruidDataSource();
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl("jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true");
dataSource.setUsername("root");
dataSource.setPassword("121212");
dataSource.setInitialSize(1);
dataSource.setMinIdle(1);
dataSource.setMaxActive(3);
dataSource.setMaxWait(60000);
dataSource.setTimeBetweenEvictionRunsMillis(60000);
dataSource.setMinEvictableIdleTimeMillis(300000);
dataSource.setPoolPreparedStatements(false);
dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
dataSource.setValidationQuery("select 1");
try {
dataSource.init();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
client/src/test/java/com/alibaba/otter/canal/client/running/kafka/KafkaClientRunningTest.java
-1
View File
@@ -11,7 +11,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
import com.alibaba.otter.canal.client.kafka.KafkaCanalConnectors;
import com.alibaba.otter.canal.protocol.Message;
/**