From 1dc534311374195b4dcc4d0c7564721c3ba78558 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2020 04:48:28 +0000 Subject: [PATCH 001/114] Bump junit from 4.11 to 4.13.1 in /elasticsearchwriter Bumps [junit](https://github.com/junit-team/junit4) from 4.11 to 4.13.1. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/main/doc/ReleaseNotes4.11.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.11...r4.13.1) Signed-off-by: dependabot[bot] --- elasticsearchwriter/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearchwriter/pom.xml b/elasticsearchwriter/pom.xml index 2a24680..a60dbd8 100644 --- a/elasticsearchwriter/pom.xml +++ b/elasticsearchwriter/pom.xml @@ -50,7 +50,7 @@ junit junit - 4.11 + 4.13.1 test From 787421fa8b775db0b8e6d68af91a9ad04d5a193f Mon Sep 17 00:00:00 2001 From: wz497345846 <497345846@qq.com> Date: Tue, 24 Aug 2021 16:05:15 +0800 Subject: [PATCH 002/114] Update mongodbwriter.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 错别字 --- mongodbwriter/doc/mongodbwriter.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodbwriter/doc/mongodbwriter.md b/mongodbwriter/doc/mongodbwriter.md index e30008d..74de8a0 100644 --- a/mongodbwriter/doc/mongodbwriter.md +++ b/mongodbwriter/doc/mongodbwriter.md @@ -139,7 +139,7 @@ MongoDBWriter通过Datax框架获取Reader生成的数据，然后将Datax支持 * splitter：特殊分隔符，当且仅当要处理的字符串要用分隔符分隔为字符数组时，才使用这个参数，通过这个参数指定的分隔符，将字符串分隔存储到MongoDB的数组中。【选填】 * upsertInfo：指定了传输数据时更新的信息。【选填】 * isUpsert：当设置为true时，表示针对相同的upsertKey做更新操作。【选填】 -* upsertKey：upsertKey指定了没行记录的业务主键。用来做更新时使用。【选填】 +* upsertKey：upsertKey指定了每行记录的业务主键。用来做更新时使用。【选填】 #### 5 类型转换 @@ -154,4 +154,4 @@ MongoDBWriter通过Datax框架获取Reader生成的数据，然后将Datax支持 #### 6 性能报告 -#### 7 测试报告 \ No newline at end of file +#### 7 测试报告 From 38b4dc8946178eb013f002f93de99a748950cd9f Mon Sep 17 00:00:00 2001 From: wang Date: Thu, 9 Sep 2021 16:31:47 +0800 Subject: [PATCH 003/114] =?UTF-8?q?fix=20bug:=20=E8=A7=A3=E5=86=B3ob=20wri?= =?UTF-8?q?ter=E8=BF=9E=E6=8E=A5=E6=95=B0=E9=9A=8F=E7=9D=80=E5=86=99?= =?UTF-8?q?=E5=85=A5=E4=B8=8D=E6=96=AD=E4=B8=8A=E6=B6=A8=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/plugin/writer/oceanbasev10writer/task/InsertTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java index 522d91a..36fc726 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java @@ -65,6 +65,7 @@ public class InsertTask implements Runnable { this.writeRecordSql = writeRecordSql; this.isStop = false; this.deleteMeta = deleteMeta; + connHolder.initConnection(); } void setWriterTask(ConcurrentTableWriterTask writerTask) { @@ -151,7 +152,6 @@ public class InsertTask implements Runnable { public void doMultiInsert(final List buffer, final boolean printCost, final long restrict) { checkMemstore(); - connHolder.initConnection(); Connection conn = connHolder.getConn(); boolean success = false; long cost = 0; From 5b975c55a775e0a1075d1076987939d1e326030e Mon Sep 17 00:00:00 2001 From: wang Date: Thu, 16 Sep 2021 19:27:56 +0800 Subject: [PATCH 004/114] =?UTF-8?q?fix=20bug=EF=BC=9A=E5=9C=A8=E5=87=BA?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E7=9A=84=E6=97=B6=E5=80=99=EF=BC=8C=E4=BC=9A?= =?UTF-8?q?=E8=B0=83=E4=B8=80=E6=AC=A1reconnect=EF=BC=8C=E8=BF=99=E9=87=8C?= =?UTF-8?q?=E9=9D=A2=E9=87=8A=E6=94=BE=E6=97=A7=E8=BF=9E=E6=8E=A5=EF=BC=8C?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E6=96=B0=E8=BF=9E=E6=8E=A5=EF=BC=9B=E8=BF=9B?= =?UTF-8?q?=E5=85=A5=E4=B8=8B=E4=B8=80=E6=AC=A1=E5=BE=AA=E7=8E=AF=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E5=80=99=EF=BC=8C=E5=8F=88=E5=88=9B=E5=BB=BA=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=E8=BF=9E=E6=8E=A5=EF=BC=8C=E6=89=80=E4=BB=A5=E5=9C=A8?= =?UTF-8?q?reconnect=E9=87=8C=E9=9D=A2=E5=88=9B=E5=BB=BA=E7=9A=84=E9=82=A3?= =?UTF-8?q?=E4=B8=AA=E6=96=B0=E8=BF=9E=E6=8E=A5=E5=B0=B1=E6=B3=84=E9=9C=B2?= =?UTF-8?q?=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/plugin/writer/oceanbasev10writer/task/InsertTask.java | 1 - 1 file changed, 1 deletion(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java index 36fc726..968908c 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/InsertTask.java @@ -165,7 +165,6 @@ public class InsertTask implements Runnable { } catch (InterruptedException e) { LOG.info("thread interrupted ..., ignore"); } - connHolder.initConnection(); conn = connHolder.getConn(); LOG.info("retry {}, start do batch insert, size={}", i, buffer.size()); checkMemstore(); From c0351b8d76a704b63fdcbbb294c57b4b47ca491a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E7=85=A6?= Date: Sat, 18 Sep 2021 19:51:03 +0800 Subject: [PATCH 005/114] dbname with minus('-') can not be parsed in ob writer --- .../plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java index 49e5c05..5988cea 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java @@ -31,7 +31,7 @@ public class ServerConnectInfo { } private void parseJdbcUrl(final String jdbcUrl) { - Pattern pattern = Pattern.compile("//([\\w\\.\\-]+:\\d+)/([\\w]+)\\?"); + Pattern pattern = Pattern.compile("//([\\w\\.\\-]+:\\d+)/([\\w-]+)\\?"); Matcher matcher = pattern.matcher(jdbcUrl); if (matcher.find()) { String ipPort = matcher.group(1); From 09374d259686bb37e533d12f5b616febf87314a8 Mon Sep 17 00:00:00 2001 From: Longda Feng Date: Fri, 24 Sep 2021 11:52:46 +0800 Subject: [PATCH 006/114] Add OceanBase supporting list. --- README.md | 3 ++- introduction.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 681de04..37a2102 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # DataX -DataX 是阿里云 [DataWorks数据集成](https://www.aliyun.com/product/bigdata/ide) 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX 实现了包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、Hologres、DRDS 等各种异构数据源之间高效的数据同步功能。 +DataX 是阿里云 [DataWorks数据集成](https://www.aliyun.com/product/bigdata/ide) 的开源版本，在阿里巴巴集团内被广泛使用的离线数据同步工具/平台。DataX 实现了包括 MySQL、Oracle、OceanBase、SqlServer、Postgre、HDFS、Hive、ADS、HBase、TableStore(OTS)、MaxCompute(ODPS)、Hologres、DRDS 等各种异构数据源之间高效的数据同步功能。 # DataX 商业版本 阿里云DataWorks数据集成是DataX团队在阿里云上的商业化产品，致力于提供复杂网络环境下、丰富的异构数据源之间高速稳定的数据移动能力，以及繁杂业务背景下的数据同步解决方案。目前已经支持云上近3000家客户，单日同步数据超过3万亿条。DataWorks数据集成目前支持离线50+种数据源，可以进行整库迁移、批量上云、增量同步、分库分表等各类同步解决方案。2020年更新实时同步能力，2020年更新实时同步能力，支持10+种数据源的读写任意组合。提供MySQL，Oracle等多种数据源到阿里云MaxCompute，Hologres等大数据引擎的一键全增量同步解决方案。 @@ -39,6 +39,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N | ------------ | ---------- | :-------: | :-------: |:-------: | | RDBMS 关系型数据库 | MySQL | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/mysqlreader/doc/mysqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md)| |             | Oracle     |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/oraclereader/doc/oraclereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/oraclewriter/doc/oraclewriter.md)| +|             | OceanBase  |     √     |     √     |[读](https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/use-datax-to-full-migration-data-to-oceanbase) 、[写](https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/use-datax-to-full-migration-data-to-oceanbase)| | | SQLServer | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/sqlserverreader/doc/sqlserverreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/sqlserverwriter/doc/sqlserverwriter.md)| | | PostgreSQL | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/postgresqlreader/doc/postgresqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/postgresqlwriter/doc/postgresqlwriter.md)| | | DRDS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/drdsreader/doc/drdsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/drdswriter/doc/drdswriter.md)| diff --git a/introduction.md b/introduction.md index b27607c..d08ad98 100644 --- a/introduction.md +++ b/introduction.md @@ -36,6 +36,7 @@ DataX本身作为离线数据同步框架，采用Framework + plugin架构构建 | ------------ | ---------- | :-------: | :-------: |:-------: | | RDBMS 关系型数据库 | MySQL | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/mysqlreader/doc/mysqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/mysqlwriter/doc/mysqlwriter.md)| |             | Oracle     |     √     |     √     |[读](https://github.com/alibaba/DataX/blob/master/oraclereader/doc/oraclereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/oraclewriter/doc/oraclewriter.md)| +|             | OceanBase  |     √     |     √     |[读](https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/use-datax-to-full-migration-data-to-oceanbase) 、[写](https://open.oceanbase.com/docs/community/oceanbase-database/V3.1.0/use-datax-to-full-migration-data-to-oceanbase)| | | SQLServer | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/sqlserverreader/doc/sqlserverreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/sqlserverwriter/doc/sqlserverwriter.md)| | | PostgreSQL | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/postgresqlreader/doc/postgresqlreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/postgresqlwriter/doc/postgresqlwriter.md)| | | DRDS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/drdsreader/doc/drdsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/drdswriter/doc/drdswriter.md)| From 26d1f11c1b55203325810bb62a94c9bd5a258469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=9F=E7=85=A6?= Date: Fri, 24 Sep 2021 18:03:38 +0800 Subject: [PATCH 007/114] obwriter: adapted for short jdbc url --- .../OceanBaseV10Writer.java | 1 - .../ext/ServerConnectInfo.java | 27 ++++++++++++++++--- .../task/ConcurrentTableWriterTask.java | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java index 89ef1c5..4ffaffe 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java @@ -20,7 +20,6 @@ import com.alibaba.datax.plugin.rdbms.writer.Constant; import com.alibaba.datax.plugin.rdbms.writer.Key; import com.alibaba.datax.plugin.rdbms.writer.util.WriterUtil; import com.alibaba.datax.plugin.writer.oceanbasev10writer.task.ConcurrentTableWriterTask; -import com.alibaba.datax.plugin.writer.oceanbasev10writer.task.SingleTableWriterTask; import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.ObWriterUtils; /** diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java index 5988cea..b061164 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/ext/ServerConnectInfo.java @@ -24,8 +24,26 @@ public class ServerConnectInfo { this.tenantName = ss[1].trim().split(":")[1]; this.jdbcUrl = ss[2].replace("jdbc:mysql:", "jdbc:oceanbase:"); } else { - throw new RuntimeException ("jdbc url format is not correct: " + jdbcUrl); + this.jdbcUrl = jdbcUrl.replace("jdbc:mysql:", "jdbc:oceanbase:"); + if (username.contains("@") && username.contains("#")) { + this.userName = username.substring(0, username.indexOf("@")); + this.tenantName = username.substring(username.indexOf("@") + 1, username.indexOf("#")); + this.clusterName = username.substring(username.indexOf("#") + 1); + } else if (username.contains(":")) { + String[] config = username.split(":"); + if (config.length != 3) { + throw new RuntimeException ("username format is not correct: " + username); + } + this.clusterName = config[0]; + this.tenantName = config[1]; + this.userName = config[2]; + } else { + this.clusterName = null; + this.tenantName = null; + this.userName = username; + } } + this.password = password; parseJdbcUrl(jdbcUrl); } @@ -51,8 +69,11 @@ public class ServerConnectInfo { } public String getFullUserName() { - StringBuilder builder = new StringBuilder(); - builder.append(userName).append("@").append(tenantName).append("#").append(clusterName); + StringBuilder builder = new StringBuilder(userName); + if (tenantName != null && clusterName != null) { + builder.append("@").append(tenantName).append("#").append(clusterName); + } + return builder.toString(); } } diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index 084acbe..cbc9a93 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -150,7 +150,7 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { partCalculator = new ObPartitionIdCalculator(connectInfo.ipPort, tableEntryKey); } catch (Exception ex) { ++retry; - LOG.warn("create new part calculator failed, retry ... {}", retry, ex); + LOG.warn("create new part calculator failed, retry {}: {}", retry, ex.getMessage()); } } while (partCalculator == null && retry < 3); // try 3 times } From 26df387f0fbb0eb1cdabc1752a09823f984b65aa Mon Sep 17 00:00:00 2001 From: hexsmith Date: Fri, 24 Sep 2021 21:13:08 +0800 Subject: [PATCH 008/114] Update mongodbreader.md --- mongodbreader/doc/mongodbreader.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongodbreader/doc/mongodbreader.md b/mongodbreader/doc/mongodbreader.md index 6d1e6a9..b61493e 100644 --- a/mongodbreader/doc/mongodbreader.md +++ b/mongodbreader/doc/mongodbreader.md @@ -8,7 +8,7 @@ MongoDBReader 插件利用 MongoDB 的java客户端MongoClient进行MongoDB的 MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的JOB程序按照指定的规则对MongoDB中的数据进行分片，并行读取，然后将MongoDB支持的类型通过逐一判断转换成Datax支持的类型。 #### 3 功能说明 -* 该示例从ODPS读一份数据到MongoDB。 +* 该示例从MongoDB读一份数据到ODPS。 { "job": { @@ -147,4 +147,4 @@ MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的J #### 6 性能报告 -#### 7 测试报告 \ No newline at end of file +#### 7 测试报告 From 7cfd0e36215fcbc7f4d8854421ab3f980754f33c Mon Sep 17 00:00:00 2001 From: hexsmith Date: Fri, 24 Sep 2021 21:23:09 +0800 Subject: [PATCH 009/114] Update hbase11xwriter.md --- hbase11xwriter/doc/hbase11xwriter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md index ff20abe..02fd550 100644 --- a/hbase11xwriter/doc/hbase11xwriter.md +++ b/hbase11xwriter/doc/hbase11xwriter.md @@ -202,7 +202,7 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba * 描述：要写入的hbase字段。index：指定该列对应reader端column的索引，从0开始；name：指定hbase表中的列，必须为 列族:列名 的格式；type：指定写入数据类型，用于转换HBase byte[]。配置格式如下： - ``` +``` "column": [ { "index":1, From 2632974c524bda68ac827e2dd1b6694ad9a58cdb Mon Sep 17 00:00:00 2001 From: hexsmith Date: Fri, 24 Sep 2021 21:23:59 +0800 Subject: [PATCH 010/114] Update hbase11xwriter.md --- hbase11xwriter/doc/hbase11xwriter.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md index 02fd550..cc519b2 100644 --- a/hbase11xwriter/doc/hbase11xwriter.md +++ b/hbase11xwriter/doc/hbase11xwriter.md @@ -203,6 +203,7 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba * 描述：要写入的hbase字段。index：指定该列对应reader端column的索引，从0开始；name：指定hbase表中的列，必须为 列族:列名 的格式；type：指定写入数据类型，用于转换HBase byte[]。配置格式如下： ``` + "column": [ { "index":1, @@ -216,7 +217,7 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba } ］ - ``` +``` * 必选：是
From 8e37018d78d2bd76a23cbf51183b72db59912f7a Mon Sep 17 00:00:00 2001 From: hexsmith Date: Fri, 24 Sep 2021 21:25:35 +0800 Subject: [PATCH 011/114] Update hbase11xwriter.md --- hbase11xwriter/doc/hbase11xwriter.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md index cc519b2..300cbfa 100644 --- a/hbase11xwriter/doc/hbase11xwriter.md +++ b/hbase11xwriter/doc/hbase11xwriter.md @@ -219,9 +219,9 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba ``` - * 必选：是
+* 必选：是
- * 默认值：无
+* 默认值：无
* **rowkeyColumn** From a080664e1ae379b922296f90e1ded3f6101ee413 Mon Sep 17 00:00:00 2001 From: hexsmith Date: Fri, 24 Sep 2021 21:26:43 +0800 Subject: [PATCH 012/114] Update hbase11xwriter.md --- hbase11xwriter/doc/hbase11xwriter.md | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md index 300cbfa..f44a610 100644 --- a/hbase11xwriter/doc/hbase11xwriter.md +++ b/hbase11xwriter/doc/hbase11xwriter.md @@ -202,26 +202,26 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba * 描述：要写入的hbase字段。index：指定该列对应reader端column的索引，从0开始；name：指定hbase表中的列，必须为 列族:列名 的格式；type：指定写入数据类型，用于转换HBase byte[]。配置格式如下： -``` + ``` -"column": [ - { - "index":1, - "name": "cf1:q1", - "type": "string" - }, - { - "index":2, - "name": "cf1:q2", - "type": "string" - } - ］ - -``` + "column": [ + { + "index":1, + "name": "cf1:q1", + "type": "string" + }, + { + "index":2, + "name": "cf1:q2", + "type": "string" + } + ］ -* 必选：是
+ ``` + + * 必选：是
-* 默认值：无
+ * 默认值：无
* **rowkeyColumn** From 133269e69e890bfa337685ecf797dd2ff035c338 Mon Sep 17 00:00:00 2001 From: hexsmith Date: Fri, 24 Sep 2021 21:28:05 +0800 Subject: [PATCH 013/114] Update hbase11xwriter.md --- hbase11xwriter/doc/hbase11xwriter.md | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/hbase11xwriter/doc/hbase11xwriter.md b/hbase11xwriter/doc/hbase11xwriter.md index f44a610..969f2e4 100644 --- a/hbase11xwriter/doc/hbase11xwriter.md +++ b/hbase11xwriter/doc/hbase11xwriter.md @@ -228,17 +228,17 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba * 描述：要写入的hbase的rowkey列。index：指定该列对应reader端column的索引，从0开始，若为常量index为－1；type：指定写入数据类型，用于转换HBase byte[]；value：配置常量，常作为多个字段的拼接符。hbasewriter会将rowkeyColumn中所有列按照配置顺序进行拼接作为写入hbase的rowkey，不能全为常量。配置格式如下： ``` -"rowkeyColumn": [ - { - "index":0, - "type":"string" - }, - { - "index":-1, - "type":"string", - "value":"_" - } - ] + "rowkeyColumn": [ + { + "index":0, + "type":"string" + }, + { + "index":-1, + "type":"string", + "value":"_" + } + ] ``` @@ -251,19 +251,19 @@ HbaseWriter 插件实现了从向Hbase中写取数据。在底层实现上，Hba * 描述：指定写入hbase的时间戳。支持：当前时间、指定时间列，指定时间，三者选一。若不配置表示用当前时间。index：指定对应reader端column的索引，从0开始，需保证能转换为long,若是Date类型，会尝试用yyyy-MM-dd HH:mm:ss和yyyy-MM-dd HH:mm:ss SSS去解析；若为指定时间index为－1；value：指定时间的值,long值。配置格式如下： ``` -"versionColumn":{ - "index":1 -} + "versionColumn":{ + "index":1 + } ``` 或者 ``` -"versionColumn":{ - "index":－1, - "value":123456789 -} + "versionColumn":{ + "index":－1, + "value":123456789 + } ``` From 65104b79eded393a25795ca4c7c06c021ceea06a Mon Sep 17 00:00:00 2001 From: PeineLiang Date: Thu, 30 Sep 2021 11:46:18 +0800 Subject: [PATCH 014/114] Update json syntax error --- drdsreader/doc/drdsreader.md | 2 +- postgresqlreader/doc/postgresqlreader.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drdsreader/doc/drdsreader.md b/drdsreader/doc/drdsreader.md index 25df920..c54e6bd 100644 --- a/drdsreader/doc/drdsreader.md +++ b/drdsreader/doc/drdsreader.md @@ -50,7 +50,7 @@ DRDS的插件目前DataX只适配了Mysql引擎的场景，DRDS对于DataX而言 // 数据库连接密码 "password": "root", "column": [ - "id"，"name" + "id","name" ], "connection": [ { diff --git a/postgresqlreader/doc/postgresqlreader.md b/postgresqlreader/doc/postgresqlreader.md index fed2c7e..02c354a 100644 --- a/postgresqlreader/doc/postgresqlreader.md +++ b/postgresqlreader/doc/postgresqlreader.md @@ -48,7 +48,7 @@ PostgresqlReader插件实现了从PostgreSQL读取数据。在底层实现上， // 数据库连接密码 "password": "xx", "column": [ - "id"，"name" + "id","name" ], //切分主键 "splitPk": "id", From d2ab6127546004569dfd5533a344a516131a618d Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 12 Oct 2021 15:44:24 +0800 Subject: [PATCH 015/114] add tdengine writer plugin --- .../com/alibaba/datax/core/EngineTest.java | 18 +++ job/opentsdb2stream.json | 31 +++++ job/opentsdb2tdengine.json | 34 ++++++ package.xml | 7 ++ pom.xml | 92 +++++++------- tdenginewriter/pom.xml | 75 ++++++++++++ tdenginewriter/src/main/assembly/package.xml | 34 ++++++ .../datax/plugin/writer/JniConnection.java | 83 +++++++++++++ .../datax/plugin/writer/TDengineWriter.java | 113 ++++++++++++++++++ .../writer/TDengineWriterErrorCode.java | 31 +++++ ...libaba_datax_plugin_writer_JniConnection.h | 87 ++++++++++++++ tdenginewriter/src/main/resources/plugin.json | 9 ++ .../main/resources/plugin_job_template.json | 10 ++ .../plugin/writer/JniConnectionTest.java | 19 +++ 14 files changed, 598 insertions(+), 45 deletions(-) create mode 100644 core/src/test/java/com/alibaba/datax/core/EngineTest.java create mode 100644 job/opentsdb2stream.json create mode 100644 job/opentsdb2tdengine.json create mode 100644 tdenginewriter/pom.xml create mode 100755 tdenginewriter/src/main/assembly/package.xml create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h create mode 100755 tdenginewriter/src/main/resources/plugin.json create mode 100644 tdenginewriter/src/main/resources/plugin_job_template.json create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java diff --git a/core/src/test/java/com/alibaba/datax/core/EngineTest.java b/core/src/test/java/com/alibaba/datax/core/EngineTest.java new file mode 100644 index 0000000..3f36263 --- /dev/null +++ b/core/src/test/java/com/alibaba/datax/core/EngineTest.java @@ -0,0 +1,18 @@ +package com.alibaba.datax.core; + + +public class EngineTest { + + public static void main(String[] args) { + System.out.println(System.getProperty("java.library.path")); +// String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "/Users/yangzy/workspace/DataX/job/opentsdb2stream.json"}; + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "/Users/yangzy/workspace/DataX/job/opentsdb2tdengine.json"}; + System.setProperty("datax.home", "/Users/yangzy/workspace/DataX/target/datax/datax"); + try { + Engine.entry(params); + } catch (Throwable e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/job/opentsdb2stream.json b/job/opentsdb2stream.json new file mode 100644 index 0000000..1ea4320 --- /dev/null +++ b/job/opentsdb2stream.json @@ -0,0 +1,31 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather.temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/job/opentsdb2tdengine.json b/job/opentsdb2tdengine.json new file mode 100644 index 0000000..cdf2277 --- /dev/null +++ b/job/opentsdb2tdengine.json @@ -0,0 +1,34 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather.temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.56.105", + "port": 6030, + "db": "test", + "user": "root", + "password": "taosdata" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/package.xml b/package.xml index 882dd23..a93e945 100755 --- a/package.xml +++ b/package.xml @@ -238,6 +238,13 @@ datax + + tdenginewriter/target/datax/ + + **/*.* + + datax + otswriter/target/datax/ diff --git a/pom.xml b/pom.xml index 3bd75a3..cb635ad 100644 --- a/pom.xml +++ b/pom.xml @@ -47,66 +47,68 @@ transformer - mysqlreader - drdsreader - sqlserverreader - postgresqlreader - kingbaseesreader - oraclereader + + + + + + odpsreader - otsreader - otsstreamreader + + txtfilereader - hdfsreader + streamreader - ossreader - ftpreader - mongodbreader + + + rdbmsreader - hbase11xreader - hbase094xreader - tsdbreader + + + opentsdbreader - cassandrareader - gdbreader - oceanbasev10reader + + + - mysqlwriter - drdswriter + + odpswriter txtfilewriter - ftpwriter - hdfswriter + + streamwriter - otswriter - oraclewriter - sqlserverwriter - postgresqlwriter - kingbaseeswriter - osswriter - mongodbwriter + + + + + + + adswriter - ocswriter + rdbmswriter - hbase11xwriter - hbase094xwriter - hbase11xsqlwriter - hbase11xsqlreader - elasticsearchwriter - tsdbwriter - adbpgwriter - gdbwriter - cassandrawriter - clickhousewriter - oscarwriter - oceanbasev10writer + + + + + + + + + + + + + plugin-rdbms-util plugin-unstructured-storage-util - hbase20xsqlreader - hbase20xsqlwriter - kuduwriter + + + + tdenginewriter diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml new file mode 100644 index 0000000..7cef9ac --- /dev/null +++ b/tdenginewriter/pom.xml @@ -0,0 +1,75 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + com.alibaba.datax.tdenginewriter + tdenginewriter + 1.0.0 + + + 8 + 8 + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + + junit + junit + ${junit-version} + test + + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + + \ No newline at end of file diff --git a/tdenginewriter/src/main/assembly/package.xml b/tdenginewriter/src/main/assembly/package.xml new file mode 100755 index 0000000..f7a7d0b --- /dev/null +++ b/tdenginewriter/src/main/assembly/package.xml @@ -0,0 +1,34 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/writer/tdenginewriter + + + target/ + + tdenginewriter-1.0.0.jar + + plugin/writer/tdenginewriter + + + + + + false + plugin/writer/tdenginewriter/libs + runtime + + + diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java new file mode 100644 index 0000000..5cbbf2a --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java @@ -0,0 +1,83 @@ +package com.alibaba.datax.plugin.writer; + +import java.util.Properties; + +public class JniConnection { + + private static final long JNI_NULL_POINTER = 0L; + private static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; + private static final String PROPERTY_KEY_LOCALE = "locale"; + private static final String PROPERTY_KEY_CHARSET = "charset"; + private static final String PROPERTY_KEY_TIME_ZONE = "timezone"; + + private long psql; + + static { + System.loadLibrary("taos"); + } + + public JniConnection(Properties props) { + if (this.psql != JNI_NULL_POINTER) { + close(); + this.psql = JNI_NULL_POINTER; + } + + initImp(props.getProperty(PROPERTY_KEY_CONFIG_DIR, null)); + + String locale = props.getProperty(PROPERTY_KEY_LOCALE); + if (setOptions(0, locale) < 0) { + throw new RuntimeException("Failed to set locale: " + locale + ". System default will be used."); + } + String charset = props.getProperty(PROPERTY_KEY_CHARSET); + if (setOptions(1, charset) < 0) { + throw new RuntimeException("Failed to set charset: " + charset + ". System default will be used."); + } + String timezone = props.getProperty(PROPERTY_KEY_TIME_ZONE); + if (setOptions(2, timezone) < 0) { + throw new RuntimeException("Failed to set timezone: " + timezone + ". System default will be used."); + } + } + + public long open(String host, int port, String dbname, String user, String password) { + if (this.psql != JNI_NULL_POINTER) { + close(); + this.psql = JNI_NULL_POINTER; + } + + this.psql = connectImp(host, port, dbname, user, password); + if (this.psql == JNI_NULL_POINTER) { + String errMsg = getErrMsgImp(0); + throw new RuntimeException(errMsg); + } + return this.psql; + } + + public void close() { + int code = this.closeConnectionImp(this.psql); + if (code != 0) { + throw new RuntimeException("JNI closeConnection failed"); + } + this.psql = JNI_NULL_POINTER; + } + + private static native void initImp(String configDir); + + private static native int setOptions(int optionIndex, String optionValue); + + private static native String getTsCharset(); + + private native long connectImp(String host, int port, String dbName, String user, String password); + + private native long executeQueryImp(byte[] sqlBytes, long connection); + + private native int getErrCodeImp(long connection, long pSql); + + private native String getErrMsgImp(long pSql); + + private native int getAffectedRowsImp(long connection, long pSql); + + private native int closeConnectionImp(long connection); + + private native long insertOpentsdbJson(String json, long pSql); + +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java new file mode 100644 index 0000000..184279e --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -0,0 +1,113 @@ +package com.alibaba.datax.plugin.writer; + + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.spi.Writer; +import com.alibaba.datax.common.util.Configuration; + +import java.io.BufferedWriter; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +public class TDengineWriter extends Writer { + + private static final String HOST = "host"; + private static final String PORT = "port"; + private static final String DBNAME = "dbname"; + private static final String USER = "user"; + private static final String PASSWORD = "password"; + + public static class Job extends Writer.Job { + + private Configuration originalConfig; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + } + + @Override + public void destroy() { + + } + + @Override + public List split(int mandatoryNumber) { + List writerSplitConfigs = new ArrayList(); + for (int i = 0; i < mandatoryNumber; i++) { + writerSplitConfigs.add(this.originalConfig); + } + + return writerSplitConfigs; + } + } + + public static class Task extends Writer.Task { + private static final String NEWLINE_FLAG = System.getProperty("line.separator", "\n"); + private Configuration writerSliceConfig; + private String peerPluginName; + + @Override + public void init() { + this.writerSliceConfig = getPluginJobConf(); + this.peerPluginName = getPeerPluginName(); + } + + @Override + public void destroy() { + + } + + @Override + public void startWrite(RecordReceiver lineReceiver) { + + String host = this.writerSliceConfig.getString(HOST); + int port = this.writerSliceConfig.getInt(PORT); + String dbname = this.writerSliceConfig.getString(DBNAME); + String user = this.writerSliceConfig.getString(USER); + String password = this.writerSliceConfig.getString(PASSWORD); + + JniConnection connection = new JniConnection(new Properties()); + long psql = connection.open(host, port, dbname, user, password); + System.out.println("psql: " + psql); + connection.close(); + + try { + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)); + + Record record; + while ((record = lineReceiver.getFromReader()) != null) { + writer.write(recordToString(record)); + } + writer.flush(); + + } catch (Exception e) { + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); + } + } + + private String recordToString(Record record) { + int recordLength = record.getColumnNumber(); + if (0 == recordLength) { + return NEWLINE_FLAG; + } + + Column column; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < recordLength; i++) { + column = record.getColumn(i); + sb.append(column.asString()).append("\t"); + } + sb.setLength(sb.length() - 1); + sb.append(NEWLINE_FLAG); + + return sb.toString(); + } + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java new file mode 100644 index 0000000..02e8707 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java @@ -0,0 +1,31 @@ +package com.alibaba.datax.plugin.writer; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum TDengineWriterErrorCode implements ErrorCode { + RUNTIME_EXCEPTION("TDengineWriter-00", "运行时异常"); + + private final String code; + private final String description; + + private TDengineWriterErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h new file mode 100644 index 0000000..0a161b9 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h @@ -0,0 +1,87 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_alibaba_datax_plugin_writer_JniConnection */ + +#ifndef _Included_com_alibaba_datax_plugin_writer_JniConnection +#define _Included_com_alibaba_datax_plugin_writer_JniConnection +#ifdef __cplusplus +extern "C" { +#endif +#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER +#define com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER 0LL +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: initImp + * Signature: (Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_initImp + (JNIEnv *, jclass, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: setOptions + * Signature: (ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_setOptions + (JNIEnv *, jclass, jint, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getTsCharset + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getTsCharset + (JNIEnv *, jclass); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: connectImp + * Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_connectImp + (JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: executeQueryImp + * Signature: ([BJ)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_executeQueryImp + (JNIEnv *, jobject, jbyteArray, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrCodeImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrCodeImp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrMsgImp + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgImp + (JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getAffectedRowsImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getAffectedRowsImp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: closeConnectionImp + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_closeConnectionImp + (JNIEnv *, jobject, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/tdenginewriter/src/main/resources/plugin.json b/tdenginewriter/src/main/resources/plugin.json new file mode 100755 index 0000000..6c900a1 --- /dev/null +++ b/tdenginewriter/src/main/resources/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "tdenginewriter", + "class": "com.alibaba.datax.plugin.writer.TDengineWriter", + "description": { + "useScene": "data migration to tdengine", + "mechanism": "use JNI to write data to tdengine." + }, + "developer": "zyyang-taosdata" +} \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/plugin_job_template.json b/tdenginewriter/src/main/resources/plugin_job_template.json new file mode 100644 index 0000000..5482b26 --- /dev/null +++ b/tdenginewriter/src/main/resources/plugin_job_template.json @@ -0,0 +1,10 @@ +{ + "name": "tdenginewriter", + "parameter": { + "host": "", + "port": 6030, + "db": "", + "user": "", + "password": "" + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java new file mode 100644 index 0000000..1c9f426 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java @@ -0,0 +1,19 @@ +package com.alibaba.datax.plugin.writer; + +import org.junit.Test; + +import java.util.Properties; + +public class JniConnectionTest { + + @Test + public void test() { + JniConnection connection = new JniConnection(new Properties()); + + long psql = connection.open("192.168.56.107", 6030, "log", "root", "taosdata"); + System.out.println("psql: " + psql); + + connection.close(); + } + +} \ No newline at end of file From 5da6e34d5c877fbf4e9d57365912cfdffca5113e Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 12 Oct 2021 15:59:45 +0800 Subject: [PATCH 016/114] add maven plugin for junit test --- tdenginewriter/pom.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 7cef9ac..2c294b8 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -69,6 +69,23 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + + + **/*Test.java + + + + + true + + + From ff87a6bcff21f3fac853111bdee8cf4f0ac11464 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 12 Oct 2021 17:47:08 +0800 Subject: [PATCH 017/114] change test case --- .../com/alibaba/datax/plugin/writer/JniConnection.java | 10 ++++++++-- .../alibaba/datax/plugin/writer/TDengineWriter.java | 5 ----- .../com_alibaba_datax_plugin_writer_JniConnection.h | 8 ++++++++ .../alibaba/datax/plugin/writer/JniConnectionTest.java | 6 ++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java index 5cbbf2a..6c533ed 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java @@ -38,7 +38,7 @@ public class JniConnection { } } - public long open(String host, int port, String dbname, String user, String password) { + public void open(String host, int port, String dbname, String user, String password) { if (this.psql != JNI_NULL_POINTER) { close(); this.psql = JNI_NULL_POINTER; @@ -49,7 +49,13 @@ public class JniConnection { String errMsg = getErrMsgImp(0); throw new RuntimeException(errMsg); } - return this.psql; + } + + public long insertOpentsdbJson(String json) { + if (this.psql == JNI_NULL_POINTER) { + throw new RuntimeException("JNI connection is NULL"); + } + return insertOpentsdbJson(json, this.psql); } public void close() { diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java index 184279e..ee080f5 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -73,11 +73,6 @@ public class TDengineWriter extends Writer { String user = this.writerSliceConfig.getString(USER); String password = this.writerSliceConfig.getString(PASSWORD); - JniConnection connection = new JniConnection(new Properties()); - long psql = connection.open(host, port, dbname, user, password); - System.out.println("psql: " + psql); - connection.close(); - try { BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h index 0a161b9..58cac66 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h @@ -81,6 +81,14 @@ JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getAff JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_closeConnectionImp (JNIEnv *, jobject, jlong); +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: insertOpentsdbJson + * Signature: (Ljava/lang/String;J)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_insertOpentsdbJson + (JNIEnv *, jobject, jstring, jlong); + #ifdef __cplusplus } #endif diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java index 1c9f426..87fa668 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java @@ -10,8 +10,10 @@ public class JniConnectionTest { public void test() { JniConnection connection = new JniConnection(new Properties()); - long psql = connection.open("192.168.56.107", 6030, "log", "root", "taosdata"); - System.out.println("psql: " + psql); + connection.open("192.168.56.105", 6030, "log", "root", "taosdata"); + + String json = "{\"metric\":\"weather.temperature\",\"timestamp\":1609430400000,\"value\":123,\"tags\":{\"location\":\"beijing\",\"id\":123}}"; + connection.insertOpentsdbJson(json); connection.close(); } From f6520cf06b4475ba2e36690e84e7af2979781552 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 13 Oct 2021 12:47:31 +0800 Subject: [PATCH 018/114] change --- .../datax/plugin/writer/JniConnection.java | 31 ++++++++++++------- ...libaba_datax_plugin_writer_JniConnection.h | 10 ++++++ .../plugin/writer/JniConnectionTest.java | 2 +- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java index 6c533ed..5512aaf 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java @@ -5,21 +5,22 @@ import java.util.Properties; public class JniConnection { private static final long JNI_NULL_POINTER = 0L; + private static final int JNI_SUCCESSFUL = 0; private static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; private static final String PROPERTY_KEY_LOCALE = "locale"; private static final String PROPERTY_KEY_CHARSET = "charset"; private static final String PROPERTY_KEY_TIME_ZONE = "timezone"; - private long psql; + private long conn; static { System.loadLibrary("taos"); } public JniConnection(Properties props) { - if (this.psql != JNI_NULL_POINTER) { + if (this.conn != JNI_NULL_POINTER) { close(); - this.psql = JNI_NULL_POINTER; + this.conn = JNI_NULL_POINTER; } initImp(props.getProperty(PROPERTY_KEY_CONFIG_DIR, null)); @@ -39,31 +40,35 @@ public class JniConnection { } public void open(String host, int port, String dbname, String user, String password) { - if (this.psql != JNI_NULL_POINTER) { + if (this.conn != JNI_NULL_POINTER) { close(); - this.psql = JNI_NULL_POINTER; + this.conn = JNI_NULL_POINTER; } - this.psql = connectImp(host, port, dbname, user, password); - if (this.psql == JNI_NULL_POINTER) { + this.conn = connectImp(host, port, dbname, user, password); + if (this.conn == JNI_NULL_POINTER) { String errMsg = getErrMsgImp(0); throw new RuntimeException(errMsg); } } - public long insertOpentsdbJson(String json) { - if (this.psql == JNI_NULL_POINTER) { + public void insertOpentsdbJson(String json) { + if (this.conn == JNI_NULL_POINTER) { throw new RuntimeException("JNI connection is NULL"); } - return insertOpentsdbJson(json, this.psql); + long code = insertOpentsdbJson(json, this.conn); + if (code != JNI_SUCCESSFUL) { + String errMsg = getErrMsgByCode(code); + throw new RuntimeException(errMsg); + } } public void close() { - int code = this.closeConnectionImp(this.psql); + int code = this.closeConnectionImp(this.conn); if (code != 0) { throw new RuntimeException("JNI closeConnection failed"); } - this.psql = JNI_NULL_POINTER; + this.conn = JNI_NULL_POINTER; } private static native void initImp(String configDir); @@ -80,6 +85,8 @@ public class JniConnection { private native String getErrMsgImp(long pSql); + private native String getErrMsgByCode(long code); + private native int getAffectedRowsImp(long connection, long pSql); private native int closeConnectionImp(long connection); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h index 58cac66..4bdf363 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h @@ -9,6 +9,8 @@ extern "C" { #endif #undef com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER #define com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER 0LL +#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL +#define com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL 0L /* * Class: com_alibaba_datax_plugin_writer_JniConnection * Method: initImp @@ -65,6 +67,14 @@ JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErr JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgImp (JNIEnv *, jobject, jlong); +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrMsgByCode + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgByCode + (JNIEnv *, jobject, jlong); + /* * Class: com_alibaba_datax_plugin_writer_JniConnection * Method: getAffectedRowsImp diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java index 87fa668..603931e 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java @@ -10,7 +10,7 @@ public class JniConnectionTest { public void test() { JniConnection connection = new JniConnection(new Properties()); - connection.open("192.168.56.105", 6030, "log", "root", "taosdata"); + connection.open("192.168.56.105", 6030, "test", "root", "taosdata"); String json = "{\"metric\":\"weather.temperature\",\"timestamp\":1609430400000,\"value\":123,\"tags\":{\"location\":\"beijing\",\"id\":123}}"; connection.insertOpentsdbJson(json); From 975096b676dc408ba311e4f74eda06876702c605 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 13 Oct 2021 17:02:43 +0800 Subject: [PATCH 019/114] change --- .../com/alibaba/datax/core/EngineTest.java | 5 +- job/opentsdb2tdengine.json | 2 +- .../datax/plugin/writer/JniConnection.java | 30 +++++----- .../datax/plugin/writer/TDengineWriter.java | 59 +++++++++++++++---- .../plugin/writer/JniConnectionTest.java | 4 +- 5 files changed, 69 insertions(+), 31 deletions(-) diff --git a/core/src/test/java/com/alibaba/datax/core/EngineTest.java b/core/src/test/java/com/alibaba/datax/core/EngineTest.java index 3f36263..47ce268 100644 --- a/core/src/test/java/com/alibaba/datax/core/EngineTest.java +++ b/core/src/test/java/com/alibaba/datax/core/EngineTest.java @@ -1,9 +1,12 @@ package com.alibaba.datax.core; +import org.junit.Test; + public class EngineTest { - public static void main(String[] args) { + @Test + public void test() { System.out.println(System.getProperty("java.library.path")); // String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "/Users/yangzy/workspace/DataX/job/opentsdb2stream.json"}; String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "/Users/yangzy/workspace/DataX/job/opentsdb2tdengine.json"}; diff --git a/job/opentsdb2tdengine.json b/job/opentsdb2tdengine.json index cdf2277..92c0f8e 100644 --- a/job/opentsdb2tdengine.json +++ b/job/opentsdb2tdengine.json @@ -7,7 +7,7 @@ "parameter": { "endpoint": "http://192.168.1.180:4242", "column": [ - "weather.temperature" + "weather_temperature" ], "beginDateTime": "2021-01-01 00:00:00", "endDateTime": "2021-01-01 01:00:00" diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java index 5512aaf..a8739c5 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java @@ -6,10 +6,10 @@ public class JniConnection { private static final long JNI_NULL_POINTER = 0L; private static final int JNI_SUCCESSFUL = 0; - private static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; - private static final String PROPERTY_KEY_LOCALE = "locale"; - private static final String PROPERTY_KEY_CHARSET = "charset"; - private static final String PROPERTY_KEY_TIME_ZONE = "timezone"; + public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; + public static final String PROPERTY_KEY_LOCALE = "locale"; + public static final String PROPERTY_KEY_CHARSET = "charset"; + public static final String PROPERTY_KEY_TIME_ZONE = "timezone"; private long conn; @@ -17,7 +17,7 @@ public class JniConnection { System.loadLibrary("taos"); } - public JniConnection(Properties props) { + public JniConnection(Properties props) throws Exception { if (this.conn != JNI_NULL_POINTER) { close(); this.conn = JNI_NULL_POINTER; @@ -27,19 +27,19 @@ public class JniConnection { String locale = props.getProperty(PROPERTY_KEY_LOCALE); if (setOptions(0, locale) < 0) { - throw new RuntimeException("Failed to set locale: " + locale + ". System default will be used."); + throw new Exception("Failed to set locale: " + locale + ". System default will be used."); } String charset = props.getProperty(PROPERTY_KEY_CHARSET); if (setOptions(1, charset) < 0) { - throw new RuntimeException("Failed to set charset: " + charset + ". System default will be used."); + throw new Exception("Failed to set charset: " + charset + ". System default will be used."); } String timezone = props.getProperty(PROPERTY_KEY_TIME_ZONE); if (setOptions(2, timezone) < 0) { - throw new RuntimeException("Failed to set timezone: " + timezone + ". System default will be used."); + throw new Exception("Failed to set timezone: " + timezone + ". System default will be used."); } } - public void open(String host, int port, String dbname, String user, String password) { + public void open(String host, int port, String dbname, String user, String password) throws Exception { if (this.conn != JNI_NULL_POINTER) { close(); this.conn = JNI_NULL_POINTER; @@ -48,25 +48,25 @@ public class JniConnection { this.conn = connectImp(host, port, dbname, user, password); if (this.conn == JNI_NULL_POINTER) { String errMsg = getErrMsgImp(0); - throw new RuntimeException(errMsg); + throw new Exception(errMsg); } } - public void insertOpentsdbJson(String json) { + public void insertOpentsdbJson(String json) throws Exception { if (this.conn == JNI_NULL_POINTER) { - throw new RuntimeException("JNI connection is NULL"); + throw new Exception("JNI connection is NULL"); } long code = insertOpentsdbJson(json, this.conn); if (code != JNI_SUCCESSFUL) { String errMsg = getErrMsgByCode(code); - throw new RuntimeException(errMsg); + throw new Exception(errMsg); } } - public void close() { + public void close() throws Exception { int code = this.closeConnectionImp(this.conn); if (code != 0) { - throw new RuntimeException("JNI closeConnection failed"); + throw new Exception("JNI closeConnection failed"); } this.conn = JNI_NULL_POINTER; } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java index ee080f5..28fa4ca 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -7,12 +7,12 @@ import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.BufferedWriter; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Properties; public class TDengineWriter extends Writer { @@ -22,6 +22,7 @@ public class TDengineWriter extends Writer { private static final String DBNAME = "dbname"; private static final String USER = "user"; private static final String PASSWORD = "password"; + private static final String PEER_PLUGIN_NAME = "peerPluginName"; public static class Job extends Writer.Job { @@ -30,6 +31,7 @@ public class TDengineWriter extends Writer { @Override public void init() { this.originalConfig = super.getPluginJobConf(); + this.originalConfig.set(PEER_PLUGIN_NAME, getPeerPluginName()); } @Override @@ -49,14 +51,14 @@ public class TDengineWriter extends Writer { } public static class Task extends Writer.Task { + private static final Logger LOG = LoggerFactory.getLogger(Job.class); + private static final String NEWLINE_FLAG = System.getProperty("line.separator", "\n"); private Configuration writerSliceConfig; - private String peerPluginName; @Override public void init() { this.writerSliceConfig = getPluginJobConf(); - this.peerPluginName = getPeerPluginName(); } @Override @@ -67,22 +69,57 @@ public class TDengineWriter extends Writer { @Override public void startWrite(RecordReceiver lineReceiver) { + String host = this.writerSliceConfig.getString(HOST); int port = this.writerSliceConfig.getInt(PORT); String dbname = this.writerSliceConfig.getString(DBNAME); String user = this.writerSliceConfig.getString(USER); String password = this.writerSliceConfig.getString(PASSWORD); - try { - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8)); + Properties properties = new Properties(); + String cfgdir = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_CONFIG_DIR); + if (cfgdir != null && !cfgdir.isEmpty()) { + properties.setProperty(JniConnection.PROPERTY_KEY_CONFIG_DIR, cfgdir); + } + String timezone = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_TIME_ZONE); + if (timezone != null && !timezone.isEmpty()) { + properties.setProperty(JniConnection.PROPERTY_KEY_TIME_ZONE, timezone); + } + String locale = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_LOCALE); + if (locale != null && !locale.isEmpty()) { + properties.setProperty(JniConnection.PROPERTY_KEY_LOCALE, locale); + } + String charset = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_CHARSET); + if (charset != null && !charset.isEmpty()) { + properties.setProperty(JniConnection.PROPERTY_KEY_CHARSET, charset); + } + String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); + if (peerPluginName.equals("opentsdbreader")) { + try { + JniConnection conn = new JniConnection(properties); + conn.open(host, port, dbname, user, password); + LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); + writeOpentsdb(lineReceiver, conn); + conn.close(); + LOG.info("TDengine connection closed"); + } catch (Exception e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + } + } + } + + private void writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn) { + try { Record record; while ((record = lineReceiver.getFromReader()) != null) { - writer.write(recordToString(record)); + String jsonData = recordToString(record); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); } - writer.flush(); - } catch (Exception e) { + LOG.error("TDengineWriter ERROR: " + e.getMessage()); throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); } } @@ -92,7 +129,6 @@ public class TDengineWriter extends Writer { if (0 == recordLength) { return NEWLINE_FLAG; } - Column column; StringBuilder sb = new StringBuilder(); for (int i = 0; i < recordLength; i++) { @@ -101,7 +137,6 @@ public class TDengineWriter extends Writer { } sb.setLength(sb.length() - 1); sb.append(NEWLINE_FLAG); - return sb.toString(); } } diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java index 603931e..040cf34 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java @@ -7,12 +7,12 @@ import java.util.Properties; public class JniConnectionTest { @Test - public void test() { + public void test() throws Exception { JniConnection connection = new JniConnection(new Properties()); connection.open("192.168.56.105", 6030, "test", "root", "taosdata"); - String json = "{\"metric\":\"weather.temperature\",\"timestamp\":1609430400000,\"value\":123,\"tags\":{\"location\":\"beijing\",\"id\":123}}"; + String json = "{\"metric\":\"weather_temperature\",\"timestamp\":1609430400000,\"value\":123,\"tags\":{\"location\":\"beijing\",\"id\":\"t123\"}}"; connection.insertOpentsdbJson(json); connection.close(); From 9121b7d3c59b0c079aabe69596ad9b01b9f3ed66 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 13 Oct 2021 17:12:02 +0800 Subject: [PATCH 020/114] change test cases --- {job => core/src/main/job}/opentsdb2stream.json | 0 {job => core/src/main/job}/opentsdb2tdengine.json | 0 core/src/test/java/com/alibaba/datax/core/EngineTest.java | 5 ++--- 3 files changed, 2 insertions(+), 3 deletions(-) rename {job => core/src/main/job}/opentsdb2stream.json (100%) rename {job => core/src/main/job}/opentsdb2tdengine.json (100%) diff --git a/job/opentsdb2stream.json b/core/src/main/job/opentsdb2stream.json similarity index 100% rename from job/opentsdb2stream.json rename to core/src/main/job/opentsdb2stream.json diff --git a/job/opentsdb2tdengine.json b/core/src/main/job/opentsdb2tdengine.json similarity index 100% rename from job/opentsdb2tdengine.json rename to core/src/main/job/opentsdb2tdengine.json diff --git a/core/src/test/java/com/alibaba/datax/core/EngineTest.java b/core/src/test/java/com/alibaba/datax/core/EngineTest.java index 47ce268..bbc5bdc 100644 --- a/core/src/test/java/com/alibaba/datax/core/EngineTest.java +++ b/core/src/test/java/com/alibaba/datax/core/EngineTest.java @@ -8,9 +8,8 @@ public class EngineTest { @Test public void test() { System.out.println(System.getProperty("java.library.path")); -// String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "/Users/yangzy/workspace/DataX/job/opentsdb2stream.json"}; - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "/Users/yangzy/workspace/DataX/job/opentsdb2tdengine.json"}; - System.setProperty("datax.home", "/Users/yangzy/workspace/DataX/target/datax/datax"); + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/opentsdb2tdengine.json"}; + System.setProperty("datax.home", "../target/datax/datax"); try { Engine.entry(params); } catch (Throwable e) { From c5d64c00a9c99812737698e0edf1fcbeddf0b134 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 13 Oct 2021 17:21:36 +0800 Subject: [PATCH 021/114] change --- core/src/main/job/opentsdb2tdengine.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/job/opentsdb2tdengine.json b/core/src/main/job/opentsdb2tdengine.json index 92c0f8e..dd33220 100644 --- a/core/src/main/job/opentsdb2tdengine.json +++ b/core/src/main/job/opentsdb2tdengine.json @@ -18,7 +18,7 @@ "parameter": { "host": "192.168.56.105", "port": 6030, - "db": "test", + "dbname": "test", "user": "root", "password": "taosdata" } From 08cc6dbdca60d1f193614b5de9ecac95bc42828d Mon Sep 17 00:00:00 2001 From: wxpang <90431887+wxpang@users.noreply.github.com> Date: Fri, 15 Oct 2021 09:31:29 +0800 Subject: [PATCH 022/114] Update PostgresqlWriter.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加bigserial的类型转换 --- .../datax/plugin/writer/postgresqlwriter/PostgresqlWriter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/postgresqlwriter/src/main/java/com/alibaba/datax/plugin/writer/postgresqlwriter/PostgresqlWriter.java b/postgresqlwriter/src/main/java/com/alibaba/datax/plugin/writer/postgresqlwriter/PostgresqlWriter.java index 22dc0c1..2d38db3 100755 --- a/postgresqlwriter/src/main/java/com/alibaba/datax/plugin/writer/postgresqlwriter/PostgresqlWriter.java +++ b/postgresqlwriter/src/main/java/com/alibaba/datax/plugin/writer/postgresqlwriter/PostgresqlWriter.java @@ -67,6 +67,8 @@ public class PostgresqlWriter extends Writer { public String calcValueHolder(String columnType){ if("serial".equalsIgnoreCase(columnType)){ return "?::int"; + }else if("bigserial".equalsIgnoreCase(columnType)){ + return "?::int8"; }else if("bit".equalsIgnoreCase(columnType)){ return "?::bit varying"; } From 0d5c8fc765b8a362f085e70826d7934447f6a8c9 Mon Sep 17 00:00:00 2001 From: kongxiaohan Date: Sat, 16 Oct 2021 18:10:51 +0800 Subject: [PATCH 023/114] modified a mistake description for rdbmsreader --- rdbmsreader/doc/rdbmsreader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdbmsreader/doc/rdbmsreader.md b/rdbmsreader/doc/rdbmsreader.md index dd3039e..fd8ae10 100644 --- a/rdbmsreader/doc/rdbmsreader.md +++ b/rdbmsreader/doc/rdbmsreader.md @@ -138,7 +138,7 @@ RDBMSReader插件实现了从RDBMS读取数据。在底层实现上，RDBMSReade **rdbmswriter如何增加新的数据库支持:** - - 进入rdbmsreader对应目录，这里${DATAX_HOME}为DataX主目录，即: ${DATAX_HOME}/plugin/reader/rdbmswriter + - 进入rdbmsreader对应目录，这里${DATAX_HOME}为DataX主目录，即: ${DATAX_HOME}/plugin/reader/rdbmsreader - 在rdbmsreader插件目录下有plugin.json配置文件，在此文件中注册您具体的数据库驱动，具体放在drivers数组中。rdbmsreader插件在任务执行时会动态选择合适的数据库驱动连接数据库。 From 5e54aee590f7ad5970b28c011d8c0e34d00c09ab Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 19 Oct 2021 15:40:11 +0800 Subject: [PATCH 024/114] tdenginewriter support batch write --- core/src/main/job/opentsdb2tdengine.json | 3 +- tdenginewriter/doc/tdenginewriter.md | 151 ++++++++++++++++++ .../datax/plugin/writer/JniConnection.java | 5 - .../com/alibaba/datax/plugin/writer/Key.java | 11 ++ .../datax/plugin/writer/TDengineWriter.java | 77 ++++----- 5 files changed, 205 insertions(+), 42 deletions(-) create mode 100644 tdenginewriter/doc/tdenginewriter.md create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java diff --git a/core/src/main/job/opentsdb2tdengine.json b/core/src/main/job/opentsdb2tdengine.json index dd33220..377b98c 100644 --- a/core/src/main/job/opentsdb2tdengine.json +++ b/core/src/main/job/opentsdb2tdengine.json @@ -20,7 +20,8 @@ "port": 6030, "dbname": "test", "user": "root", - "password": "taosdata" + "password": "taosdata", + "batchSize": 1000 } } } diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md new file mode 100644 index 0000000..8e55b18 --- /dev/null +++ b/tdenginewriter/doc/tdenginewriter.md @@ -0,0 +1,151 @@ +# DataX TDengineWriter + +## 1 快速介绍 + +TDengineWriter 插件实现了写入数据到 TDengine 的功能。 在底层实现上， TDengineWriter 通过 JNI的方式调用libtaos.so/tao.dll中的方法，连接 TDengine +数据库实例，并执行schemaless的写入。 TDengineWriter 面向ETL开发工程师，他们使用 TDengineWriter 从数仓导入数据到 TDengine。同时，TDengineWriter +亦可以作为数据迁移工具为DBA等用户提供服务。 + +## 2 实现原理 + +TDengineWriter 通过 DataX 框架获取 Reader +生成的协议数据，根据reader的类型解析数据，通过JNI方式调用libtaos.so（或taos.dll）中的方法，使用schemaless的方式写入到TDengine。 + +## 3 功能说明 + +### 3.1 配置样例 + +* 这里使用一份从OpenTSDB产生到 TDengine 导入的数据。 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather_temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.1.180", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +### 3.2 参数说明 + +* **host** + * 描述：TDengine实例的host。 + + * 必选：是
+ + * 默认值：无
+* **port** + * 描述：TDengine实例的port。 + * 必选：是
+ * 默认值：无
+* **dbname** + * 描述：目的数据库的名称。 + + * 必选：是
+ + * 默认值：无
+* **username** + * 描述：TDengine实例的用户名
+ * 必选：是
+ * 默认值：无
+* **password** + * 描述：TDengine实例的密码
+ * 必选：是
+ * 默认值：无
+ +### 3.3 类型转换 + +目前，由于opentsdbreader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： + +| OpenTSDB数据类型 | DataX 内部类型| TDengine 数据类型 | +| -------- | ----- | -------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String |binary | +| String（tag） | String |binary | + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡流出流量(MB/s)|DataX机器运行负载|DB网卡进入流量(MB/s)|DB运行负载|DB TPS| +|--------| --------|--------|--------|--------|--------|--------|--------| +|1| | | | | | | | +|4| | | | | | | | +|8| | | | | | | | +|16| | | | | | | | +|32| | | | | | | | + +说明： + +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 +3. 16通道，4096批量提交时，出现 full gc 2次。 + +#### 4.2.4 性能测试小结 + +1. +2. + +## 5 约束限制 + +## FAQ \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java index a8739c5..b167063 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java @@ -18,11 +18,6 @@ public class JniConnection { } public JniConnection(Properties props) throws Exception { - if (this.conn != JNI_NULL_POINTER) { - close(); - this.conn = JNI_NULL_POINTER; - } - initImp(props.getProperty(PROPERTY_KEY_CONFIG_DIR, null)); String locale = props.getProperty(PROPERTY_KEY_LOCALE); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java new file mode 100644 index 0000000..b240bce --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java @@ -0,0 +1,11 @@ +package com.alibaba.datax.plugin.writer; + +public class Key { + public static final String HOST = "host"; + public static final String PORT = "port"; + public static final String DBNAME = "dbname"; + public static final String USER = "user"; + public static final String PASSWORD = "password"; + public static final String BATCH_SIZE = "batchSize"; + +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java index 28fa4ca..60c7652 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -10,19 +10,12 @@ import com.alibaba.datax.common.util.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Properties; +import java.util.*; public class TDengineWriter extends Writer { - private static final String HOST = "host"; - private static final String PORT = "port"; - private static final String DBNAME = "dbname"; - private static final String USER = "user"; - private static final String PASSWORD = "password"; private static final String PEER_PLUGIN_NAME = "peerPluginName"; + private static final String DEFAULT_BATCH_SIZE = "1"; public static class Job extends Writer.Job { @@ -45,7 +38,6 @@ public class TDengineWriter extends Writer { for (int i = 0; i < mandatoryNumber; i++) { writerSplitConfigs.add(this.originalConfig); } - return writerSplitConfigs; } } @@ -68,53 +60,66 @@ public class TDengineWriter extends Writer { @Override public void startWrite(RecordReceiver lineReceiver) { - - - String host = this.writerSliceConfig.getString(HOST); - int port = this.writerSliceConfig.getInt(PORT); - String dbname = this.writerSliceConfig.getString(DBNAME); - String user = this.writerSliceConfig.getString(USER); - String password = this.writerSliceConfig.getString(PASSWORD); - + Set keys = this.writerSliceConfig.getKeys(); Properties properties = new Properties(); - String cfgdir = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_CONFIG_DIR); - if (cfgdir != null && !cfgdir.isEmpty()) { - properties.setProperty(JniConnection.PROPERTY_KEY_CONFIG_DIR, cfgdir); - } - String timezone = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_TIME_ZONE); - if (timezone != null && !timezone.isEmpty()) { - properties.setProperty(JniConnection.PROPERTY_KEY_TIME_ZONE, timezone); - } - String locale = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_LOCALE); - if (locale != null && !locale.isEmpty()) { - properties.setProperty(JniConnection.PROPERTY_KEY_LOCALE, locale); - } - String charset = this.writerSliceConfig.getString(JniConnection.PROPERTY_KEY_CHARSET); - if (charset != null && !charset.isEmpty()) { - properties.setProperty(JniConnection.PROPERTY_KEY_CHARSET, charset); + for (String key : keys) { + String value = this.writerSliceConfig.getString(key); + properties.setProperty(key, value); } String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); if (peerPluginName.equals("opentsdbreader")) { + // opentsdb json protocol use JNI and schemaless API to write + + String host = properties.getProperty(Key.HOST); + int port = Integer.parseInt(properties.getProperty(Key.PORT)); + String dbname = properties.getProperty(Key.DBNAME); + String user = properties.getProperty(Key.USER); + String password = properties.getProperty(Key.PASSWORD); + try { JniConnection conn = new JniConnection(properties); conn.open(host, port, dbname, user, password); LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); - writeOpentsdb(lineReceiver, conn); + + int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, DEFAULT_BATCH_SIZE)); + writeOpentsdb(lineReceiver, conn, batchSize); conn.close(); LOG.info("TDengine connection closed"); } catch (Exception e) { LOG.error(e.getMessage()); e.printStackTrace(); } + } else { + // other } } - private void writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn) { + private void writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn, int batchSize) { try { Record record; + StringBuilder sb = new StringBuilder(); + long recordIndex = 1; while ((record = lineReceiver.getFromReader()) != null) { - String jsonData = recordToString(record); + if (batchSize == 1) { + String jsonData = recordToString(record); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + } else if (recordIndex % batchSize == 1) { + sb.append("[").append(recordToString(record)).append(","); + } else if (recordIndex % batchSize == 0) { + sb.append(recordToString(record)).append("]"); + String jsonData = sb.toString(); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + sb.delete(0, sb.length()); + } else { + sb.append(recordToString(record)).append(","); + } + recordIndex++; + } + if (sb.length() != 0 && sb.charAt(0) == '[') { + String jsonData = sb.deleteCharAt(sb.length() - 1).append("]").toString(); LOG.debug(">>> " + jsonData); conn.insertOpentsdbJson(jsonData); } From ab526ca5c4d4c846277224cb122fec581d3f368c Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 19 Oct 2021 15:58:47 +0800 Subject: [PATCH 025/114] tdenginewriter --- .../datax/plugin/writer/TDengineWriter.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java index 60c7652..9dc42d9 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -45,7 +45,6 @@ public class TDengineWriter extends Writer { public static class Task extends Writer.Task { private static final Logger LOG = LoggerFactory.getLogger(Job.class); - private static final String NEWLINE_FLAG = System.getProperty("line.separator", "\n"); private Configuration writerSliceConfig; @Override @@ -70,25 +69,30 @@ public class TDengineWriter extends Writer { String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); if (peerPluginName.equals("opentsdbreader")) { // opentsdb json protocol use JNI and schemaless API to write - String host = properties.getProperty(Key.HOST); int port = Integer.parseInt(properties.getProperty(Key.PORT)); String dbname = properties.getProperty(Key.DBNAME); String user = properties.getProperty(Key.USER); String password = properties.getProperty(Key.PASSWORD); + JniConnection conn = null; try { - JniConnection conn = new JniConnection(properties); + conn = new JniConnection(properties); conn.open(host, port, dbname, user, password); LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); - int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, DEFAULT_BATCH_SIZE)); writeOpentsdb(lineReceiver, conn, batchSize); - conn.close(); - LOG.info("TDengine connection closed"); } catch (Exception e) { LOG.error(e.getMessage()); e.printStackTrace(); + } finally { + try { + if (conn != null) + conn.close(); + } catch (Exception e) { + e.printStackTrace(); + } + LOG.info("TDengine connection closed"); } } else { // other @@ -132,7 +136,7 @@ public class TDengineWriter extends Writer { private String recordToString(Record record) { int recordLength = record.getColumnNumber(); if (0 == recordLength) { - return NEWLINE_FLAG; + return ""; } Column column; StringBuilder sb = new StringBuilder(); @@ -141,7 +145,6 @@ public class TDengineWriter extends Writer { sb.append(column.asString()).append("\t"); } sb.setLength(sb.length() - 1); - sb.append(NEWLINE_FLAG); return sb.toString(); } } From 7d9543105de02c68a26f088a13024994452c25ca Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 20 Oct 2021 14:21:50 +0800 Subject: [PATCH 026/114] data handle use stratgy pattern --- core/src/main/job/stream2tdengine.json | 56 +++++++++++ .../com/alibaba/datax/core/EngineTest.java | 3 +- .../datax/plugin/writer/DataHandler.java | 10 ++ .../plugin/writer/DataHandlerFactory.java | 10 ++ .../plugin/writer/DefaultDataHandler.java | 34 +++++++ .../plugin/writer/OpentsdbDataHandler.java | 98 +++++++++++++++++++ .../datax/plugin/writer/TDengineWriter.java | 91 ++--------------- 7 files changed, 218 insertions(+), 84 deletions(-) create mode 100644 core/src/main/job/stream2tdengine.json create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java diff --git a/core/src/main/job/stream2tdengine.json b/core/src/main/job/stream2tdengine.json new file mode 100644 index 0000000..6af6832 --- /dev/null +++ b/core/src/main/job/stream2tdengine.json @@ -0,0 +1,56 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "streamreader", + "parameter": { + "column": [ + { + "random": "2021-01-01 00:00:00, 2021-01-01 23:59:59", + "type": "date" + }, + { + "random": "0, 10000", + "type": "long" + }, + { + "random": "0, 10", + "type": "string" + }, + { + "random": "0, 5", + "type": "bool" + }, + { + "random": "0, 10", + "type": "double" + }, + { + "random": "0, 10", + "type": "bytes" + } + ], + "sliceRecordCount": 100 + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.56.105", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000 + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/core/src/test/java/com/alibaba/datax/core/EngineTest.java b/core/src/test/java/com/alibaba/datax/core/EngineTest.java index bbc5bdc..92c50a7 100644 --- a/core/src/test/java/com/alibaba/datax/core/EngineTest.java +++ b/core/src/test/java/com/alibaba/datax/core/EngineTest.java @@ -8,7 +8,8 @@ public class EngineTest { @Test public void test() { System.out.println(System.getProperty("java.library.path")); - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/opentsdb2tdengine.json"}; +// String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/opentsdb2tdengine.json"}; + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/stream2tdengine.json"}; System.setProperty("datax.home", "../target/datax/datax"); try { Engine.entry(params); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java new file mode 100644 index 0000000..94d1db3 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java @@ -0,0 +1,10 @@ +package com.alibaba.datax.plugin.writer; + +import com.alibaba.datax.common.plugin.RecordReceiver; + +import java.util.Properties; + +public interface DataHandler { + + long handle(RecordReceiver lineReceiver, Properties properties); +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java new file mode 100644 index 0000000..a488e7d --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java @@ -0,0 +1,10 @@ +package com.alibaba.datax.plugin.writer; + +public class DataHandlerFactory { + + public static DataHandler build(String peerPluginName) { + if (peerPluginName.equals("opentsdbreader")) + return new OpentsdbDataHandler(); + return new DefaultDataHandler(); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java new file mode 100644 index 0000000..a1d52d7 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java @@ -0,0 +1,34 @@ +package com.alibaba.datax.plugin.writer; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.plugin.RecordReceiver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +public class DefaultDataHandler implements DataHandler { + private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); + + @Override + public long handle(RecordReceiver lineReceiver, Properties properties) { + long count = 0; + Record record; + while ((record = lineReceiver.getFromReader()) != null) { + + int recordLength = record.getColumnNumber(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < recordLength; i++) { + Column column = record.getColumn(i); + sb.append(column.asString()).append("\t"); + } + sb.setLength(sb.length() - 1); + LOG.debug(sb.toString()); + + count++; + } + return count; + } + +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java new file mode 100644 index 0000000..599e5f3 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java @@ -0,0 +1,98 @@ +package com.alibaba.datax.plugin.writer; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +public class OpentsdbDataHandler implements DataHandler { + private static final Logger LOG = LoggerFactory.getLogger(OpentsdbDataHandler.class); + private static final String DEFAULT_BATCH_SIZE = "1"; + + @Override + public long handle(RecordReceiver lineReceiver, Properties properties) { + // opentsdb json protocol use JNI and schemaless API to write + String host = properties.getProperty(Key.HOST); + int port = Integer.parseInt(properties.getProperty(Key.PORT)); + String dbname = properties.getProperty(Key.DBNAME); + String user = properties.getProperty(Key.USER); + String password = properties.getProperty(Key.PASSWORD); + + JniConnection conn = null; + long count = 0; + try { + conn = new JniConnection(properties); + conn.open(host, port, dbname, user, password); + LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); + int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, DEFAULT_BATCH_SIZE)); + count = writeOpentsdb(lineReceiver, conn, batchSize); + } catch (Exception e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + } finally { + try { + if (conn != null) + conn.close(); + } catch (Exception e) { + e.printStackTrace(); + } + LOG.info("TDengine connection closed"); + } + + return count; + } + + private long writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn, int batchSize) { + long recordIndex = 1; + try { + Record record; + StringBuilder sb = new StringBuilder(); + while ((record = lineReceiver.getFromReader()) != null) { + if (batchSize == 1) { + String jsonData = recordToString(record); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + } else if (recordIndex % batchSize == 1) { + sb.append("[").append(recordToString(record)).append(","); + } else if (recordIndex % batchSize == 0) { + sb.append(recordToString(record)).append("]"); + String jsonData = sb.toString(); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + sb.delete(0, sb.length()); + } else { + sb.append(recordToString(record)).append(","); + } + recordIndex++; + } + if (sb.length() != 0 && sb.charAt(0) == '[') { + String jsonData = sb.deleteCharAt(sb.length() - 1).append("]").toString(); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + } + } catch (Exception e) { + LOG.error("TDengineWriter ERROR: " + e.getMessage()); + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); + } + return recordIndex - 1; + } + + private String recordToString(Record record) { + int recordLength = record.getColumnNumber(); + if (0 == recordLength) { + return ""; + } + Column column; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < recordLength; i++) { + column = record.getColumn(i); + sb.append(column.asString()).append("\t"); + } + sb.setLength(sb.length() - 1); + return sb.toString(); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java index 9dc42d9..8460080 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -1,21 +1,20 @@ package com.alibaba.datax.plugin.writer; -import com.alibaba.datax.common.element.Column; -import com.alibaba.datax.common.element.Record; -import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Set; public class TDengineWriter extends Writer { private static final String PEER_PLUGIN_NAME = "peerPluginName"; - private static final String DEFAULT_BATCH_SIZE = "1"; public static class Job extends Writer.Job { @@ -67,85 +66,11 @@ public class TDengineWriter extends Writer { } String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); - if (peerPluginName.equals("opentsdbreader")) { - // opentsdb json protocol use JNI and schemaless API to write - String host = properties.getProperty(Key.HOST); - int port = Integer.parseInt(properties.getProperty(Key.PORT)); - String dbname = properties.getProperty(Key.DBNAME); - String user = properties.getProperty(Key.USER); - String password = properties.getProperty(Key.PASSWORD); - - JniConnection conn = null; - try { - conn = new JniConnection(properties); - conn.open(host, port, dbname, user, password); - LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); - int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, DEFAULT_BATCH_SIZE)); - writeOpentsdb(lineReceiver, conn, batchSize); - } catch (Exception e) { - LOG.error(e.getMessage()); - e.printStackTrace(); - } finally { - try { - if (conn != null) - conn.close(); - } catch (Exception e) { - e.printStackTrace(); - } - LOG.info("TDengine connection closed"); - } - } else { - // other - } + LOG.debug("start to handle record from: " + peerPluginName); + DataHandler handler = DataHandlerFactory.build(peerPluginName); + long records = handler.handle(lineReceiver, properties); + LOG.debug("handle data finished, records: " + records); } - private void writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn, int batchSize) { - try { - Record record; - StringBuilder sb = new StringBuilder(); - long recordIndex = 1; - while ((record = lineReceiver.getFromReader()) != null) { - if (batchSize == 1) { - String jsonData = recordToString(record); - LOG.debug(">>> " + jsonData); - conn.insertOpentsdbJson(jsonData); - } else if (recordIndex % batchSize == 1) { - sb.append("[").append(recordToString(record)).append(","); - } else if (recordIndex % batchSize == 0) { - sb.append(recordToString(record)).append("]"); - String jsonData = sb.toString(); - LOG.debug(">>> " + jsonData); - conn.insertOpentsdbJson(jsonData); - sb.delete(0, sb.length()); - } else { - sb.append(recordToString(record)).append(","); - } - recordIndex++; - } - if (sb.length() != 0 && sb.charAt(0) == '[') { - String jsonData = sb.deleteCharAt(sb.length() - 1).append("]").toString(); - LOG.debug(">>> " + jsonData); - conn.insertOpentsdbJson(jsonData); - } - } catch (Exception e) { - LOG.error("TDengineWriter ERROR: " + e.getMessage()); - throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); - } - } - - private String recordToString(Record record) { - int recordLength = record.getColumnNumber(); - if (0 == recordLength) { - return ""; - } - Column column; - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < recordLength; i++) { - column = record.getColumn(i); - sb.append(column.asString()).append("\t"); - } - sb.setLength(sb.length() - 1); - return sb.toString(); - } } } From efa1de0513b1460795ec0905e3029ea53069756c Mon Sep 17 00:00:00 2001 From: Archie Date: Sat, 23 Oct 2021 19:20:21 +0800 Subject: [PATCH 027/114] word spelling mistake. :) --- core/src/main/java/com/alibaba/datax/core/Engine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/alibaba/datax/core/Engine.java b/core/src/main/java/com/alibaba/datax/core/Engine.java index f80d792..be21512 100755 --- a/core/src/main/java/com/alibaba/datax/core/Engine.java +++ b/core/src/main/java/com/alibaba/datax/core/Engine.java @@ -73,7 +73,7 @@ public class Engine { boolean traceEnable = allConf.getBool(CoreConstant.DATAX_CORE_CONTAINER_TRACE_ENABLE, true); boolean perfReportEnable = allConf.getBool(CoreConstant.DATAX_CORE_REPORT_DATAX_PERFLOG, true); - //standlone模式的datax shell任务不进行汇报 + //standalone模式的 datax shell任务不进行汇报 if(instanceId == -1){ perfReportEnable = false; } From cd6a2f73dae68b44724a422782d2c23f66c3ee67 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 26 Oct 2021 16:09:39 +0800 Subject: [PATCH 028/114] [TD-10725]: add tdenginereader --- core/src/main/job/mysql2tdengine.json | 46 ++++++ core/src/main/job/tdengine2tdengine.json | 38 +++++ package.xml | 41 +++-- pom.xml | 3 +- tdenginereader/doc/tdenginereader.md | 145 ++++++++++++++++++ tdenginereader/pom.xml | 90 +++++++++++ tdenginereader/src/main/assembly/package.xml | 34 ++++ .../datax/plugin/reader/TDengineReader.java | 47 ++++++ tdenginereader/src/main/resources/plugin.json | 9 ++ .../main/resources/plugin_job_template.json | 14 ++ tdenginewriter/pom.xml | 2 +- tdenginewriter/src/main/assembly/package.xml | 2 +- .../datax/plugin/writer/JniConnection.java | 18 +-- 13 files changed, 459 insertions(+), 30 deletions(-) create mode 100644 core/src/main/job/mysql2tdengine.json create mode 100644 core/src/main/job/tdengine2tdengine.json create mode 100644 tdenginereader/doc/tdenginereader.md create mode 100644 tdenginereader/pom.xml create mode 100755 tdenginereader/src/main/assembly/package.xml create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java create mode 100755 tdenginereader/src/main/resources/plugin.json create mode 100644 tdenginereader/src/main/resources/plugin_job_template.json diff --git a/core/src/main/job/mysql2tdengine.json b/core/src/main/job/mysql2tdengine.json new file mode 100644 index 0000000..530ee2b --- /dev/null +++ b/core/src/main/job/mysql2tdengine.json @@ -0,0 +1,46 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "root", + "column": [ + "id", + "name" + ], + "splitPk": "db_id", + "connection": [ + { + "table": [ + "test" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/database" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.56.105", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000 + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/core/src/main/job/tdengine2tdengine.json b/core/src/main/job/tdengine2tdengine.json new file mode 100644 index 0000000..750ae20 --- /dev/null +++ b/core/src/main/job/tdengine2tdengine.json @@ -0,0 +1,38 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "host": "192.168.1.82", + "port": 6030, + "db": "test", + "user": "root", + "password": "taosdata", + "sql": "select * from weather", + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-02 00:00:00", + "splitInterval": "1h" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.56.105", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000 + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/package.xml b/package.xml index a93e945..1b291a9 100755 --- a/package.xml +++ b/package.xml @@ -103,13 +103,13 @@ datax - - otsstreamreader/target/datax/ - - **/*.* - - datax - + + otsstreamreader/target/datax/ + + **/*.* + + datax + txtfilereader/target/datax/ @@ -138,7 +138,7 @@ datax - + ftpreader/target/datax/ **/*.* @@ -180,6 +180,13 @@ datax + + tdenginereader/target/datax/ + + **/*.* + + datax + @@ -238,13 +245,6 @@ datax - - tdenginewriter/target/datax/ - - **/*.* - - datax - otswriter/target/datax/ @@ -259,7 +259,7 @@ datax - + oraclewriter/target/datax/ **/*.* @@ -273,7 +273,7 @@ datax - + postgresqlwriter/target/datax/ **/*.* @@ -399,5 +399,12 @@ datax + + tdenginewriter/target/datax/ + + **/*.* + + datax + diff --git a/pom.xml b/pom.xml index cb635ad..2358e21 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ transformer - + mysqlreader @@ -109,6 +109,7 @@ tdenginewriter + tdenginereader diff --git a/tdenginereader/doc/tdenginereader.md b/tdenginereader/doc/tdenginereader.md new file mode 100644 index 0000000..284b8e6 --- /dev/null +++ b/tdenginereader/doc/tdenginereader.md @@ -0,0 +1,145 @@ +# DataX TDengineReader + +## 1 快速介绍 + +TDengineReader 插件实现了 TDengine 读取数据的功能。 + +## 2 实现原理 + +TDengineReader 通过TDengine的JDBC driver查询获取数据。 + +## 3 功能说明 + +### 3.1 配置样例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "host": "192.168.1.82", + "port": 6030, + "db": "test", + "user": "root", + "password": "taosdata", + "sql": "select * from weather", + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-02 00:00:00", + "splitInterval": "1h" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +### 3.2 参数说明 + +* **host** + * 描述：TDengine实例的host。 + * 必选：是
+ * 默认值：无
+* **port** + * 描述：TDengine实例的port。 + * 必选：是
+ * 默认值：无
+* **dbname** + * 描述：目的数据库的名称。 + * 必选：是
+ * 默认值：无
+* **username** + * 描述：TDengine实例的用户名
+ * 必选：是
+ * 默认值：无
+* **password** + * 描述：TDengine实例的密码
+ * 必选：是
+ * 默认值：无
+* **sql** + * 描述：用来筛选迁移数据的sql
+ * 必选：是
+ * 默认值：无
+* **beginDateTime** + * 描述：TDengine实例的密码
+ * 必选：是
+ * 默认值：无
+* **endDateTime** + * 描述：
+ * 必选：是
+ * 默认值：无
+* **splitInterval** + * 描述：按照splitInterval来划分task, 每splitInterval创建一个task
+ * 必选：否
+ * 默认值：1h
+ +### 3.3 类型转换 + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡流出流量(MB/s)|DataX机器运行负载|DB网卡进入流量(MB/s)|DB运行负载|DB TPS| +|--------| --------|--------|--------|--------|--------|--------|--------| +|1| | | | | | | | +|4| | | | | | | | +|8| | | | | | | | +|16| | | | | | | | +|32| | | | | | | | + +说明： +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 + +#### 4.2.4 性能测试小结 + +1. +2. + +## 5 约束限制 + +## FAQ \ No newline at end of file diff --git a/tdenginereader/pom.xml b/tdenginereader/pom.xml new file mode 100644 index 0000000..66c64ea --- /dev/null +++ b/tdenginereader/pom.xml @@ -0,0 +1,90 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + tdenginereader + + + 8 + 8 + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + + junit + junit + ${junit-version} + test + + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + + + **/*Test.java + + + + + true + + + + + + + \ No newline at end of file diff --git a/tdenginereader/src/main/assembly/package.xml b/tdenginereader/src/main/assembly/package.xml new file mode 100755 index 0000000..b52f20f --- /dev/null +++ b/tdenginereader/src/main/assembly/package.xml @@ -0,0 +1,34 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/tdenginereader + + + target/ + + tdenginereader-0.0.1-SNAPSHOT.jar + + plugin/reader/tdenginereader + + + + + + false + plugin/reader/tdenginereader/libs + runtime + + + diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java new file mode 100644 index 0000000..cec88ed --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -0,0 +1,47 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; + +import java.util.List; + +public class TDengineReader extends Reader { + + public static class Job extends Reader.Job { + + @Override + public void init() { + + } + + @Override + public void destroy() { + + } + + @Override + public List split(int adviceNumber) { + return null; + } + } + + public static class Task extends Reader.Task { + + @Override + public void init() { + + } + + @Override + public void destroy() { + + } + + @Override + public void startRead(RecordSender recordSender) { + + } + } + +} diff --git a/tdenginereader/src/main/resources/plugin.json b/tdenginereader/src/main/resources/plugin.json new file mode 100755 index 0000000..dc91982 --- /dev/null +++ b/tdenginereader/src/main/resources/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "tdenginereader", + "class": "com.alibaba.datax.plugin.reader.TDengineReader", + "description": { + "useScene": "data migration from tdengine", + "mechanism": "use JNI to read data from tdengine." + }, + "developer": "zyyang-taosdata" +} \ No newline at end of file diff --git a/tdenginereader/src/main/resources/plugin_job_template.json b/tdenginereader/src/main/resources/plugin_job_template.json new file mode 100644 index 0000000..3e09dff --- /dev/null +++ b/tdenginereader/src/main/resources/plugin_job_template.json @@ -0,0 +1,14 @@ +{ + "name": "tdenginereader", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "db": "test", + "user": "root", + "password": "taosdata", + "sql": "select * from weather", + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-02 00:00:00", + "splitInterval": "1h" + } +} \ No newline at end of file diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 2c294b8..d658d4a 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -11,7 +11,7 @@ com.alibaba.datax.tdenginewriter tdenginewriter - 1.0.0 + 0.0.1-SNAPSHOT 8 diff --git a/tdenginewriter/src/main/assembly/package.xml b/tdenginewriter/src/main/assembly/package.xml index f7a7d0b..d3b75ea 100755 --- a/tdenginewriter/src/main/assembly/package.xml +++ b/tdenginewriter/src/main/assembly/package.xml @@ -18,7 +18,7 @@ target/ - tdenginewriter-1.0.0.jar + tdenginewriter-0.0.1-SNAPSHOT.jar plugin/writer/tdenginewriter diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java index b167063..3ce786e 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java @@ -51,11 +51,15 @@ public class JniConnection { if (this.conn == JNI_NULL_POINTER) { throw new Exception("JNI connection is NULL"); } - long code = insertOpentsdbJson(json, this.conn); - if (code != JNI_SUCCESSFUL) { - String errMsg = getErrMsgByCode(code); + + long result = insertOpentsdbJson(json, this.conn); + int errCode = getErrCodeImp(this.conn, result); + if (errCode != JNI_SUCCESSFUL) { + String errMsg = getErrMsgImp(result); + freeResultSetImp(this.conn, result); throw new Exception(errMsg); } + freeResultSetImp(this.conn, result); } public void close() throws Exception { @@ -70,19 +74,13 @@ public class JniConnection { private static native int setOptions(int optionIndex, String optionValue); - private static native String getTsCharset(); - private native long connectImp(String host, int port, String dbName, String user, String password); - private native long executeQueryImp(byte[] sqlBytes, long connection); - private native int getErrCodeImp(long connection, long pSql); private native String getErrMsgImp(long pSql); - private native String getErrMsgByCode(long code); - - private native int getAffectedRowsImp(long connection, long pSql); + private native void freeResultSetImp(long connection, long pSql); private native int closeConnectionImp(long connection); From 072b6b4ed41721abd6c4d79c98ab8008eef158d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=80=E4=B8=AA=E4=BA=BA=E5=90=83=E7=81=AB=E9=94=85?= <40656096+OverYoung@users.noreply.github.com> Date: Tue, 2 Nov 2021 20:18:18 +0800 Subject: [PATCH 029/114] =?UTF-8?q?=E4=B8=8D=E5=BD=B1=E5=93=8D=E9=98=85?= =?UTF-8?q?=E8=AF=BB=EF=BC=8C=E5=B0=B1=E6=98=AF=E8=BF=99=E4=B8=AA=E5=AD=97?= =?UTF-8?q?=E6=89=93=E9=94=99=E4=BA=86=E7=9C=8B=E7=9D=80=E9=9A=BE=E5=8F=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mysqlreader/doc/mysqlreader.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysqlreader/doc/mysqlreader.md b/mysqlreader/doc/mysqlreader.md index 3ae52af..bae4bce 100644 --- a/mysqlreader/doc/mysqlreader.md +++ b/mysqlreader/doc/mysqlreader.md @@ -165,7 +165,7 @@ MysqlReader插件实现了从Mysql读取数据。在底层实现上，MysqlReade 支持常量配置，用户需要按照Mysql SQL语法格式: ["id", "\`table\`", "1", "'bazhen.csy'", "null", "to_char(a + 1)", "2.3" , "true"] - id为普通列名，\`table\`为包含保留在的列名，1为整形数字常量，'bazhen.csy'为字符串常量，null为空指针，to_char(a + 1)为表达式，2.3为浮点数，true为布尔值。 + id为普通列名，\`table\`为包含保留字的列名，1为整形数字常量，'bazhen.csy'为字符串常量，null为空指针，to_char(a + 1)为表达式，2.3为浮点数，true为布尔值。 * 必选：是
From 490cba2c0bec5bc2e49c321a6f2a0e1ca9fba679 Mon Sep 17 00:00:00 2001 From: Vitaly Date: Sat, 6 Nov 2021 23:17:30 +0300 Subject: [PATCH 030/114] Remove wrong link There is no OCSReader in the project. Wrong link has been removed from table. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37a2102..fe5d122 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N | 阿里云数仓数据存储 | ODPS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/odpsreader/doc/odpsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/odpswriter/doc/odpswriter.md)| | | ADS | | √ |[写](https://github.com/alibaba/DataX/blob/master/adswriter/doc/adswriter.md)| | | OSS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/ossreader/doc/ossreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/osswriter/doc/osswriter.md)| -| | OCS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/ocsreader/doc/ocsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/ocswriter/doc/ocswriter.md)| +| | OCS | | √ |[写](https://github.com/alibaba/DataX/blob/master/ocswriter/doc/ocswriter.md)| | NoSQL数据存储 | OTS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/otsreader/doc/otsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/otswriter/doc/otswriter.md)| | | Hbase0.94 | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hbase094xreader/doc/hbase094xreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase094xwriter/doc/hbase094xwriter.md)| | | Hbase1.1 | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hbase11xreader/doc/hbase11xreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hbase11xwriter/doc/hbase11xwriter.md)| From e60948b604f8887addc806160eb89984bd477333 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 11 Nov 2021 15:24:09 +0800 Subject: [PATCH 031/114] add option authDB to mongodb reader doc --- mongodbreader/doc/mongodbreader.md | 1 + 1 file changed, 1 insertion(+) diff --git a/mongodbreader/doc/mongodbreader.md b/mongodbreader/doc/mongodbreader.md index b61493e..99d2573 100644 --- a/mongodbreader/doc/mongodbreader.md +++ b/mongodbreader/doc/mongodbreader.md @@ -127,6 +127,7 @@ MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的J * address： MongoDB的数据地址信息，因为MonogDB可能是个集群，则ip端口信息需要以Json数组的形式给出。【必填】 * userName：MongoDB的用户名。【选填】 * userPassword： MongoDB的密码。【选填】 +* authDb: MongoDB认证数据库【选填】 * collectionName： MonogoDB的集合名。【必填】 * column：MongoDB的文档列名。【必填】 * name：Column的名字。【必填】 From 3de5a8f71517268afe77415f71f6b5511d81a8b6 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 11 Nov 2021 17:38:14 +0800 Subject: [PATCH 032/114] add test job configure --- core/src/main/job/mongodb2tdengine.json | 75 +++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 core/src/main/job/mongodb2tdengine.json diff --git a/core/src/main/job/mongodb2tdengine.json b/core/src/main/job/mongodb2tdengine.json new file mode 100644 index 0000000..0667bdd --- /dev/null +++ b/core/src/main/job/mongodb2tdengine.json @@ -0,0 +1,75 @@ +{ + "job": { + "setting": { + "speed": { + "channel": 2 + } + }, + "content": [ + { + "reader": { + "name": "mongodbreader", + "parameter": { + "address": [ + "123.56.104.14:27017" + ], + "userName": "admin678", + "mechanism": "SCRAM-SHA-1", + "userPassword": "huwG86123", + "authDb": "admin", + "dbName": "test", + "collectionName": "cu_market_data", + "column": [ + { + "name": "instrumentID", + "type": "string" + }, + { + "name": "tradeTime", + "type": "date" + }, + { + "name": "lastPrice", + "type": "double" + }, + { + "name": "askPrice1", + "type": "double" + }, + { + "name": "bidPrice1", + "type": "double" + }, + { + "name": "volume", + "type": "int" + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "123.56.104.14", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "measurement": "market_snapshot", + "tag_set": { + "product": "cu", + "instrumentID": 0 + }, + "field_set": { + "lastPrice": 2, + "askPrice1": 3, + "bidPrice1": 4, + "volume": 5 + }, + "timestamp": 1 + } + } + } + ] + } +} \ No newline at end of file From 906cc24ba16a8410cf935a2931b5a313c4446045 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 12 Nov 2021 15:48:24 +0800 Subject: [PATCH 033/114] [TD-10787]: migrate mysql data to tdengine --- core/src/main/job/mysql2tdengine.json | 12 +-- core/src/main/job/opentsdb2stream.json | 31 -------- .../com/alibaba/datax/core/EngineTest.java | 2 +- tdenginewriter/pom.xml | 6 ++ .../datax/plugin/writer/CommonUtil.java | 20 +++++ .../datax/plugin/writer/DataHandler.java | 5 +- .../plugin/writer/DataHandlerFactory.java | 2 + .../plugin/writer/DefaultDataHandler.java | 5 +- .../datax/plugin/writer/MysqlDataHandler.java | 73 +++++++++++++++++++ .../plugin/writer/OpentsdbDataHandler.java | 4 +- .../datax/plugin/writer/TDengineWriter.java | 20 ++--- 11 files changed, 125 insertions(+), 55 deletions(-) delete mode 100644 core/src/main/job/opentsdb2stream.json create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/CommonUtil.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/MysqlDataHandler.java diff --git a/core/src/main/job/mysql2tdengine.json b/core/src/main/job/mysql2tdengine.json index 530ee2b..7978fbf 100644 --- a/core/src/main/job/mysql2tdengine.json +++ b/core/src/main/job/mysql2tdengine.json @@ -6,19 +6,18 @@ "name": "mysqlreader", "parameter": { "username": "root", - "password": "root", + "password": "123456", "column": [ - "id", - "name" + "*" ], - "splitPk": "db_id", + "splitPk": "f1", "connection": [ { "table": [ - "test" + "weather" ], "jdbcUrl": [ - "jdbc:mysql://127.0.0.1:3306/database" + "jdbc:mysql://192.168.56.105:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" ] } ] @@ -32,6 +31,7 @@ "dbname": "test", "user": "root", "password": "taosdata", + "table": "weather", "batchSize": 1000 } } diff --git a/core/src/main/job/opentsdb2stream.json b/core/src/main/job/opentsdb2stream.json deleted file mode 100644 index 1ea4320..0000000 --- a/core/src/main/job/opentsdb2stream.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "opentsdbreader", - "parameter": { - "endpoint": "http://192.168.1.180:4242", - "column": [ - "weather.temperature" - ], - "beginDateTime": "2021-01-01 00:00:00", - "endDateTime": "2021-01-01 01:00:00" - } - }, - "writer": { - "name": "streamwriter", - "parameter": { - "encoding": "UTF-8", - "print": true - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/core/src/test/java/com/alibaba/datax/core/EngineTest.java b/core/src/test/java/com/alibaba/datax/core/EngineTest.java index 92c50a7..fe4dff8 100644 --- a/core/src/test/java/com/alibaba/datax/core/EngineTest.java +++ b/core/src/test/java/com/alibaba/datax/core/EngineTest.java @@ -9,7 +9,7 @@ public class EngineTest { public void test() { System.out.println(System.getProperty("java.library.path")); // String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/opentsdb2tdengine.json"}; - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/stream2tdengine.json"}; + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/mysql2tdengine.json"}; System.setProperty("datax.home", "../target/datax/datax"); try { Engine.entry(params); diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index d658d4a..d8df0c5 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -31,6 +31,12 @@ + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + + junit junit diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/CommonUtil.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/CommonUtil.java new file mode 100644 index 0000000..573b00f --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/CommonUtil.java @@ -0,0 +1,20 @@ +package com.alibaba.datax.plugin.writer; + +import com.alibaba.datax.common.util.Configuration; + +import java.util.Properties; +import java.util.Set; + +public class CommonUtil { + + public static Properties toProperties(Configuration configuration) { + Set keys = configuration.getKeys(); + Properties properties = new Properties(); + for (String key : keys) { + String value = configuration.getString(key); + if (value != null) + properties.setProperty(key, value); + } + return properties; + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java index 94d1db3..e7a1328 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java @@ -1,10 +1,9 @@ package com.alibaba.datax.plugin.writer; import com.alibaba.datax.common.plugin.RecordReceiver; - -import java.util.Properties; +import com.alibaba.datax.common.util.Configuration; public interface DataHandler { - long handle(RecordReceiver lineReceiver, Properties properties); + long handle(RecordReceiver lineReceiver, Configuration properties); } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java index a488e7d..26b3e47 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java @@ -5,6 +5,8 @@ public class DataHandlerFactory { public static DataHandler build(String peerPluginName) { if (peerPluginName.equals("opentsdbreader")) return new OpentsdbDataHandler(); + if (peerPluginName.equals("mysqlreader")) + return new MysqlDataHandler(); return new DefaultDataHandler(); } } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java index a1d52d7..54d32d7 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java @@ -3,16 +3,15 @@ package com.alibaba.datax.plugin.writer; import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.util.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Properties; - public class DefaultDataHandler implements DataHandler { private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); @Override - public long handle(RecordReceiver lineReceiver, Properties properties) { + public long handle(RecordReceiver lineReceiver, Configuration configuration) { long count = 0; Record record; while ((record = lineReceiver.getFromReader()) != null) { diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/MysqlDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/MysqlDataHandler.java new file mode 100644 index 0000000..fd52b77 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/MysqlDataHandler.java @@ -0,0 +1,73 @@ +package com.alibaba.datax.plugin.writer; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.util.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class MysqlDataHandler implements DataHandler { + private static final Logger LOG = LoggerFactory.getLogger(MysqlDataHandler.class); + Connection conn; + + @Override + public long handle(RecordReceiver lineReceiver, Configuration configuration) { + Properties properties = CommonUtil.toProperties(configuration); + + long count = 0; + try { + conn = getConnection(properties); + + Record record; + while ((record = lineReceiver.getFromReader()) != null) { + + int recordLength = record.getColumnNumber(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < recordLength; i++) { + Column column = record.getColumn(i); + sb.append(column.asString()).append("\t"); + } + sb.setLength(sb.length() - 1); + LOG.debug(sb.toString()); + + count++; + } + + + } finally { + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + return count; + } + + private Connection getConnection(Properties properties) { + String host = properties.getProperty(Key.HOST); + int port = Integer.parseInt(properties.getProperty(Key.PORT)); + String dbname = properties.getProperty(Key.DBNAME); + String user = properties.getProperty(Key.USER); + String password = properties.getProperty(Key.PASSWORD); + String url = "jdbc:TAOS://" + host + ":" + port + "/" + dbname + "?user=" + user + "&password=" + password; + Connection connection = null; + try { + connection = DriverManager.getConnection(url, properties); + LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); + } catch (SQLException e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + } + return connection; + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java index 599e5f3..2096909 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java @@ -4,6 +4,7 @@ import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.util.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,8 +15,9 @@ public class OpentsdbDataHandler implements DataHandler { private static final String DEFAULT_BATCH_SIZE = "1"; @Override - public long handle(RecordReceiver lineReceiver, Properties properties) { + public long handle(RecordReceiver lineReceiver, Configuration configuration) { // opentsdb json protocol use JNI and schemaless API to write + Properties properties = CommonUtil.toProperties(configuration); String host = properties.getProperty(Key.HOST); int port = Integer.parseInt(properties.getProperty(Key.PORT)); String dbname = properties.getProperty(Key.DBNAME); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java index 8460080..73087ac 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java @@ -9,13 +9,19 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; -import java.util.Properties; -import java.util.Set; public class TDengineWriter extends Writer { private static final String PEER_PLUGIN_NAME = "peerPluginName"; + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + public static class Job extends Writer.Job { private Configuration originalConfig; @@ -49,6 +55,7 @@ public class TDengineWriter extends Writer { @Override public void init() { this.writerSliceConfig = getPluginJobConf(); + } @Override @@ -58,17 +65,10 @@ public class TDengineWriter extends Writer { @Override public void startWrite(RecordReceiver lineReceiver) { - Set keys = this.writerSliceConfig.getKeys(); - Properties properties = new Properties(); - for (String key : keys) { - String value = this.writerSliceConfig.getString(key); - properties.setProperty(key, value); - } - String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); LOG.debug("start to handle record from: " + peerPluginName); DataHandler handler = DataHandlerFactory.build(peerPluginName); - long records = handler.handle(lineReceiver, properties); + long records = handler.handle(lineReceiver, writerSliceConfig); LOG.debug("handle data finished, records: " + records); } From bf01999222ff86d0275c1f7112db9846e814b729 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 18 Nov 2021 10:28:20 +0800 Subject: [PATCH 034/114] mongodb2tdengine support --- core/src/main/job/mongodb2tdengine.json | 11 +- tdenginewriter/doc/tdenginewriter.md | 260 ++++++++++++++---- tdenginewriter/pom.xml | 11 +- .../plugin/writer/DefaultDataHandler.java | 34 --- .../{ => tdenginewriter}/DataHandler.java | 2 +- .../DataHandlerFactory.java | 2 +- .../tdenginewriter/DefaultDataHandler.java | 101 +++++++ .../tdenginewriter/JDBCBatchWriter.java | 149 ++++++++++ .../{ => tdenginewriter}/JniConnection.java | 2 +- .../writer/{ => tdenginewriter}/Key.java | 7 +- .../OpentsdbDataHandler.java | 2 +- .../writer/tdenginewriter/SchemaManager.java | 255 +++++++++++++++++ .../{ => tdenginewriter}/TDengineWriter.java | 10 +- .../TDengineWriterErrorCode.java | 5 +- ...gin_writer_tdenginewriter_JniConnection.h} | 0 tdenginewriter/src/main/resources/plugin.json | 4 +- .../JniConnectionTest.java | 2 +- .../tdenginewriter/TDengineWriterTest.java | 21 ++ 18 files changed, 775 insertions(+), 103 deletions(-) delete mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/DataHandler.java (77%) rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/DataHandlerFactory.java (81%) create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/JniConnection.java (98%) rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/Key.java (52%) rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/OpentsdbDataHandler.java (98%) create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/TDengineWriter.java (84%) rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/TDengineWriterErrorCode.java (75%) rename tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/{com_alibaba_datax_plugin_writer_JniConnection.h => tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h} (100%) rename tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/{ => tdenginewriter}/JniConnectionTest.java (90%) create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java diff --git a/core/src/main/job/mongodb2tdengine.json b/core/src/main/job/mongodb2tdengine.json index 0667bdd..4cfc987 100644 --- a/core/src/main/job/mongodb2tdengine.json +++ b/core/src/main/job/mongodb2tdengine.json @@ -55,18 +55,21 @@ "dbname": "test", "user": "root", "password": "taosdata", - "measurement": "market_snapshot", - "tag_set": { + "stable": "market_snapshot", + "batchSize": 35, + "tagColumn": { "product": "cu", "instrumentID": 0 }, - "field_set": { + "fieldColumn": { "lastPrice": 2, "askPrice1": 3, "bidPrice1": 4, "volume": 5 }, - "timestamp": 1 + "timestampColumn": { + "tradeTime": 1 + } } } } diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 8e55b18..c9c222a 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -2,20 +2,21 @@ ## 1 快速介绍 -TDengineWriter 插件实现了写入数据到 TDengine 的功能。 在底层实现上， TDengineWriter 通过 JNI的方式调用libtaos.so/tao.dll中的方法，连接 TDengine -数据库实例，并执行schemaless的写入。 TDengineWriter 面向ETL开发工程师，他们使用 TDengineWriter 从数仓导入数据到 TDengine。同时，TDengineWriter -亦可以作为数据迁移工具为DBA等用户提供服务。 +TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 ## 2 实现原理 -TDengineWriter 通过 DataX 框架获取 Reader -生成的协议数据，根据reader的类型解析数据，通过JNI方式调用libtaos.so（或taos.dll）中的方法，使用schemaless的方式写入到TDengine。 +TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： + +1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 + +2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 + +这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 ## 3 功能说明 - -### 3.1 配置样例 - -* 这里使用一份从OpenTSDB产生到 TDengine 导入的数据。 +### 3.1 从OpenTSDB到TDengine +#### 3.1.1 配置样例 ```json { @@ -54,46 +55,189 @@ TDengineWriter 通过 DataX 框架获取 Reader } ``` -### 3.2 参数说明 +#### 3.1.2 参数说明 -* **host** - * 描述：TDengine实例的host。 +| 参数 | 描述 | 是否必选 | 默认值 | +| --------- | -------------------- | -------- | -------- | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbname | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1 | - * 必选：是
- * 默认值：无
-* **port** - * 描述：TDengine实例的port。 - * 必选：是
- * 默认值：无
-* **dbname** - * 描述：目的数据库的名称。 +#### 3.1.3 类型转换 - * 必选：是
+目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： - * 默认值：无
-* **username** - * 描述：TDengine实例的用户名
- * 必选：是
- * 默认值：无
-* **password** - * 描述：TDengine实例的密码
- * 必选：是
- * 默认值：无
- -### 3.3 类型转换 - -目前，由于opentsdbreader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： - -| OpenTSDB数据类型 | DataX 内部类型| TDengine 数据类型 | -| -------- | ----- | -------- | +| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | | timestamp | Date | timestamp | | Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | +| Float（value） | Double | double | +| String（value） | String | binary | | Integer（tag） | String | binary | -| Float（tag） | String |binary | -| String（tag） | String |binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | + +### 3.2 从MongoDB到TDengine + +#### 3.2.1 配置样例 +```json +{ + "job": { + "setting": { + "speed": { + "channel": 2 + } + }, + "content": [ + { + "reader": { + "name": "mongodbreader", + "parameter": { + "address": [ + "127.0.0.1:27017" + ], + "userName": "user", + "mechanism": "SCRAM-SHA-1", + "userPassword": "password", + "authDb": "admin", + "dbName": "test", + "collectionName": "stock", + "column": [ + { + "name": "stockID", + "type": "string" + }, + { + "name": "tradeTime", + "type": "date" + }, + { + "name": "lastPrice", + "type": "double" + }, + { + "name": "askPrice1", + "type": "double" + }, + { + "name": "bidPrice1", + "type": "double" + }, + { + "name": "volume", + "type": "int" + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "localhost", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "stable": "stock", + "tagColumn": { + "industry": "energy", + "stockID": 0 + }, + "fieldColumn": { + "lastPrice": 2, + "askPrice1": 3, + "bidPrice1": 4, + "volume": 5 + }, + "timestampColumn": { + "tradeTime": 1 + } + } + } + } + ] + } +} +``` + +**注：本配置的writer部分同样适用于关系型数据库** + + +#### 3.2.2 参数说明 +| 参数 | 描述 | 是否必选 | 默认值 | 备注 | +| --------------- | -------------------- | ---------------- | -------- | ------------------ | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbname | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1000 | +| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | +| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | +| fieldColumn | 字段列的列名和位置 | 否 | 无 | | +| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | + +#### 3.3.3 自动建表规则 +##### 3.3.3.1 超级表创建规则 + +如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
+数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： + +```sql +CREATE STABLE IF NOT EXISTS market_snapshot ( + tadetime TIMESTAMP, + lastprice DOUBLE, + askprice1 DOUBLE, + bidprice1 DOUBLE, + volume INT +) +TAGS( + industry NCHAR(64), + stockID NCHAR(64 +); +``` + +##### 3.3.3.2 子表创建规则 + +子表结果与超表相同，子表表名生成规则： +1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 +2. 计算该字符串的 MD5 散列值 "md5_val"。 +3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 + +#### 3.3.4 用户提前建表 + +如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 +此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： +``` + Field | Type | Length | Note | +================================================================================= + ts | TIMESTAMP | 8 | | + current | DOUBLE | 8 | | + location | BINARY | 10 | TAG | +``` +那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 + +#### 3.3.5 注意事项 + +1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 +2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 +3. 插件优先使用配置文件中指定的表结构。 + +#### 3.3.6 类型转换 + +| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| int, Long | Long | BIGINT | +| double | Double | DOUBLE | +| string, array | String | NCHAR(64) | +| date | Date | TIMESTAMP | +| boolean | Boolean | BOOL | +| bytes | Bytes | BINARY | + ## 4 性能报告 @@ -127,13 +271,13 @@ TDengineWriter 通过 DataX 框架获取 Reader #### 4.2.1 单表测试报告 -| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡流出流量(MB/s)|DataX机器运行负载|DB网卡进入流量(MB/s)|DB运行负载|DB TPS| -|--------| --------|--------|--------|--------|--------|--------|--------| -|1| | | | | | | | -|4| | | | | | | | -|8| | | | | | | | -|16| | | | | | | | -|32| | | | | | | | +| 通道数 | DataX速度(Rec/s) | DataX流量(MB/s) | DataX机器网卡流出流量(MB/s) | DataX机器运行负载 | DB网卡进入流量(MB/s) | DB运行负载 | DB TPS | +| ------ | ---------------- | --------------- | --------------------------- | ----------------- | -------------------- | ---------- | ------ | +| 1 | | | | | | | | +| 4 | | | | | | | | +| 8 | | | | | | | | +| 16 | | | | | | | | +| 32 | | | | | | | | 说明： @@ -143,9 +287,23 @@ TDengineWriter 通过 DataX 框架获取 Reader #### 4.2.4 性能测试小结 -1. -2. ## 5 约束限制 -## FAQ \ No newline at end of file +## FAQ + +### 如何选取要同步的数据的范围？ + +数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 + +### 如何一次导入多张源表？ + +如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 + +### 1张源表导入之后对应TDengine中多少张表？ + +这是又tagColumn决定的，如果所有tag列的值都相同，目标表也只有一个。源表有多少不同的tag组合，目标超表就会有多少子表。 + +### 源表和目标表的字段顺序一致吗？ + +TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表是自动调整。 \ No newline at end of file diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index d658d4a..8eb94b3 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -19,6 +19,11 @@ + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + com.alibaba.datax datax-common @@ -37,7 +42,11 @@ ${junit-version} test - + + org.apache.commons + commons-lang3 + ${commons-lang3-version} + diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java deleted file mode 100644 index a1d52d7..0000000 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DefaultDataHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.alibaba.datax.plugin.writer; - -import com.alibaba.datax.common.element.Column; -import com.alibaba.datax.common.element.Record; -import com.alibaba.datax.common.plugin.RecordReceiver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Properties; - -public class DefaultDataHandler implements DataHandler { - private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); - - @Override - public long handle(RecordReceiver lineReceiver, Properties properties) { - long count = 0; - Record record; - while ((record = lineReceiver.getFromReader()) != null) { - - int recordLength = record.getColumnNumber(); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < recordLength; i++) { - Column column = record.getColumn(i); - sb.append(column.asString()).append("\t"); - } - sb.setLength(sb.length() - 1); - LOG.debug(sb.toString()); - - count++; - } - return count; - } - -} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java similarity index 77% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java index 94d1db3..686ac27 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.plugin.RecordReceiver; diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java similarity index 81% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java index a488e7d..1f740d7 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/DataHandlerFactory.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; public class DataHandlerFactory { diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java new file mode 100644 index 0000000..733f49c --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -0,0 +1,101 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.taosdata.jdbc.TSDBPreparedStatement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +/** + * 默认DataHandler + */ +public class DefaultDataHandler implements DataHandler { + private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); + + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + @Override + public long handle(RecordReceiver lineReceiver, Properties properties) { + SchemaManager schemaManager = new SchemaManager(properties); + if (!schemaManager.configValid()) { + return 0; + } + + try { + Connection conn = getTaosConnection(properties); + if (conn == null) { + return 0; + } + if (schemaManager.shouldGuessSchema()) { + LOG.info("无法从配置文件获取表结构信息，尝试从数据库获取"); + boolean success = schemaManager.getFromDB(conn); + if (!success) { + return 0; + } + } else { + + } + int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, "1000")); + return write(lineReceiver, conn, batchSize, schemaManager); + } catch (Exception e) { + LOG.error("write failed " + e.getMessage()); + e.printStackTrace(); + } + return 0; + } + + + private Connection getTaosConnection(Properties properties) throws SQLException { + // 检查必要参数 + String host = properties.getProperty(Key.HOST); + String port = properties.getProperty(Key.PORT); + String dbname = properties.getProperty(Key.DBNAME); + String user = properties.getProperty(Key.USER); + String password = properties.getProperty(Key.PASSWORD); + if (host == null || port == null || dbname == null || user == null || password == null) { + String keys = String.join(" ", Key.HOST, Key.PORT, Key.DBNAME, Key.USER, Key.PASSWORD); + LOG.error("Required options missing, please check: " + keys); + return null; + } + String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", host, port, dbname, user, password); + LOG.info("TDengine connection established, host:{} port:{} dbname:{} user:{}", host, port, dbname, user); + return DriverManager.getConnection(jdbcUrl); + } + + /** + * 使用SQL批量写入
+ * + * @return 成功写入记录数 + * @throws SQLException + */ + private long write(RecordReceiver lineReceiver, Connection conn, int batchSize, SchemaManager scm) throws SQLException { + Record record = lineReceiver.getFromReader(); + if (record == null) { + return 0; + } + if (scm.shouldCreateTable()) { + scm.createSTable(conn, record); + } + String pq = String.format("INSERT INTO ? USING %s TAGS(%s) (%s) values (%s)", scm.getStable(), scm.getTagValuesPlaceHolder(), scm.getJoinedFieldNames(), scm.getFieldValuesPlaceHolder()); + LOG.info("Prepared SQL: {}", pq); + try (TSDBPreparedStatement stmt = (TSDBPreparedStatement) conn.prepareStatement(pq)) { + JDBCBatchWriter batchWriter = new JDBCBatchWriter(stmt, scm, batchSize); + do { + batchWriter.append(record); + } while ((record = lineReceiver.getFromReader()) != null); + batchWriter.flush(); + return batchWriter.getCount(); + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java new file mode 100644 index 0000000..17023d0 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -0,0 +1,149 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.taosdata.jdbc.TSDBPreparedStatement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * 使用JDBC原生写入接口批量写入。
+ * 有两个限制条件导致批量写入的代码逻辑过于复杂，以至于需要开发新的类来封装。
+ * 1. 用户必须提前把需要批量写入的数据搜集到ArrayList中 + * 2. 每批写入的表名必须相同。 + * 这个类的实现逻辑是： + * 1. 先把属于同一子表的Record缓存起来 + * 2. 缓存的数量达到batchSize阈值，自动执行一次批量写入 + * 3. 最后一批数据需要用户手动flush才能写入 + */ +public class JDBCBatchWriter { + public static final Logger LOG = LoggerFactory.getLogger(JDBCBatchWriter.class); + + private TSDBPreparedStatement stmt; + private SchemaManager scm; + private int batchSize; + // 缓存Record， key为tableName + Map> buf = new HashMap<>(); + // 缓存表的标签值， key为tableName + Map tableTagValues = new HashMap<>(); + private long sucCount = 0; + private final int tsColIndex; + private List fieldList; + private Map fieldIndexMap; + + public JDBCBatchWriter(TSDBPreparedStatement stmt, SchemaManager scm, int batchSize) { + this.stmt = stmt; + this.scm = scm; + this.batchSize = batchSize; + this.tsColIndex = scm.getTsColIndex(); + this.fieldList = scm.getFieldList(); + this.fieldIndexMap = scm.getFieldIndexMap(); + } + + + public void append(Record record) throws SQLException { + String[] tagValues = scm.getTagValuesFromRecord(record); + String tableName = scm.computeTableName(tagValues); + if (buf.containsKey(tableName)) { + List lis = buf.get(tableName); + lis.add(record); + if (lis.size() == batchSize) { + executeBatch(tableName); + lis.clear(); + } + } else { + List lis = new ArrayList<>(batchSize); + lis.add(record); + buf.put(tableName, lis); + tableTagValues.put(tableName, tagValues); + } + } + + /** + * 执行单表批量写入 + * + * @param tableName + * @throws SQLException + */ + private void executeBatch(String tableName) throws SQLException { + // 表名 + stmt.setTableName(tableName); + List records = buf.get(tableName); + // 标签 + String[] tagValues = tableTagValues.get(tableName); + LOG.debug("executeBatch {}", String.join(",", tagValues)); + for (int i = 0; i < tagValues.length; ++i) { + stmt.setTagNString(i, tagValues[i]); + } + // 时间戳 + ArrayList tsList = records.stream().map(r -> r.getColumn(tsColIndex).asDate().getTime()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setTimestamp(0, tsList); + // 字段 + Record record = records.get(0); + for (int i = 0; i < fieldList.size(); ) { + String fieldName = fieldList.get(i); + int index = fieldIndexMap.get(fieldName); + Column column = record.getColumn(index); + switch (column.getType()) { + case LONG: + ArrayList lisLong = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setLong(++i, lisLong); + break; + case DOUBLE: + ArrayList lisDouble = records.stream().map(r -> r.getColumn(index).asDouble()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setDouble(++i, lisDouble); + break; + case STRING: + ArrayList lisString = records.stream().map(r -> r.getColumn(index).asString()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setNString(++i, lisString, 64); + break; + case DATE: + ArrayList lisTs = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setTimestamp(++i, lisTs); + break; + case BOOL: + ArrayList lisBool = records.stream().map(r -> r.getColumn(index).asBoolean()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setBoolean(++i, lisBool); + break; + case BYTES: + ArrayList lisBytes = records.stream().map(r -> r.getColumn(index).asString()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setString(++i, lisBytes, 64); + break; + default: + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, column.getType().toString()); + } + } + // 执行 + stmt.columnDataAddBatch(); + stmt.columnDataExecuteBatch(); + // 更新计数器 + sucCount += records.size(); + } + + /** + * 把缓存的Record全部写入 + */ + public void flush() throws SQLException { + for (String tabName : buf.keySet()) { + if (buf.get(tabName).size() > 0) { + executeBatch(tabName); + } + } + stmt.columnDataCloseBatch(); + } + + /** + * @return 成功写入的数据量 + */ + public long getCount() { + return sucCount; + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java similarity index 98% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java index 3ce786e..0aabe32 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/JniConnection.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; import java.util.Properties; diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java similarity index 52% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index b240bce..090a799 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; public class Key { public static final String HOST = "host"; @@ -7,5 +7,8 @@ public class Key { public static final String USER = "user"; public static final String PASSWORD = "password"; public static final String BATCH_SIZE = "batchSize"; - + public static final String STABLE = "stable"; + public static final String TAG_COLUMN = "tagColumn"; + public static final String FIELD_COLUMN = "fieldColumn"; + public static final String TIMESTAMP_COLUMN = "timestampColumn"; } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java similarity index 98% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java index 599e5f3..52f1aa7 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/OpentsdbDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java new file mode 100644 index 0000000..b3d7b7e --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -0,0 +1,255 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; +import java.util.stream.Collectors; + +public class SchemaManager { + private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class); + + private String stable; // 目标超表名 + private Map fixedTagValue = new HashMap<>(); // 固定标签值 标签名 -> 标签值 + private Map tagIndexMap = new HashMap<>(); // 动态标签值 标签名 -> 列索引 + private Map fieldIndexMap = new HashMap<>(); // 字段名 -> 字段索引 + private String tsColName; // 时间戳列名 + private int tsColIndex = -1; // 时间戳列索引 + private List fieldList = new ArrayList<>(); + private List tagList = new ArrayList<>(); + private boolean canInferSchemaFromConfig = false; + + + public SchemaManager() { + } + + public SchemaManager(Properties properties) { + getFromConfig(properties); + } + + private String mapDataxType(Column.Type type) { + switch (type) { + case LONG: + return "BIGINT"; + case DOUBLE: + return "DOUBLE"; + case STRING: + return "NCHAR(64)"; + case DATE: + return "TIMESTAMP"; + case BOOL: + return "BOOL"; + case BYTES: + return "BINARY"; + default: + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, type.toString()); + } + } + + public void setStable(String stable) { + stable = stable; + } + + public String getStable() { + return stable; + } + + private void getFromConfig(Properties properties) { + stable = properties.getProperty(Key.STABLE); + if (stable == null) { + LOG.error("配置错误: no stable"); + return; + } + for (Object key : properties.keySet()) { + String k = (String) key; + String v = properties.getProperty(k); + + String[] ps = k.split("\\."); + if (ps.length == 1) { + continue; + } + if (k.startsWith(Key.TAG_COLUMN)) { + String tagName = ps[1]; + try { + Integer tagIndex = Integer.parseInt(v); + this.tagIndexMap.put(tagName, tagIndex); + tagList.add(tagName); + } catch (NumberFormatException e) { + fixedTagValue.put(tagName, v); + tagList.add(tagName); + } + } else if (k.startsWith(Key.FIELD_COLUMN)) { + String fieldName = ps[1]; + Integer fileIndex = Integer.parseInt(v); + fieldIndexMap.put(fieldName, fileIndex); + } else if (k.startsWith(Key.TIMESTAMP_COLUMN)) { + tsColName = ps[1]; + tsColIndex = Integer.parseInt(v); + } + } + List sortedFieldName = fieldIndexMap.entrySet().stream().sorted((x, y) -> x.getValue().compareTo(y.getValue())).map(e -> e.getKey()).collect(Collectors.toList()); + fieldList.addAll(sortedFieldName); // 排序的目的是保证自动建表时列的顺序和输入数据的列的顺序保持一致 + canInferSchemaFromConfig = tsColIndex > -1 && !(fixedTagValue.isEmpty() && tagIndexMap.isEmpty()) && !fieldIndexMap.isEmpty(); + LOG.info("配置文件解析结果：fixedTags=[{}] ,tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", fixedTagValue.keySet()), String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); + } + + public boolean shouldGuessSchema() { + return !canInferSchemaFromConfig; + } + + public boolean shouldCreateTable() { + return canInferSchemaFromConfig; + } + + public boolean configValid() { + boolean valid = (tagList.size() > 0 && fieldList.size() > 0 && tsColIndex > -1) || (tagList.size() == 0 && fieldList.size() == 0 && tsColIndex == -1); + if (!valid) { + LOG.error("配置错误. tag_columns，field_columns，timestamp_column必须同时存在或同时省略，当前解析结果: tag_columns: {}, field_columns:{}, timestamp_column:{} tsColIndex:{}", + (fixedTagValue.size() + tagIndexMap.size()), fieldIndexMap.size(), tsColName, tsColIndex); + } + return valid; + } + + /** + * 通过执行`describe dbname.stable`命令，获取表的schema.
+ * describe命名返回有4列内容，分布是：Field,Type,Length,Note
+ * + * @return 成功返回true，如果超表不存在或其他错误则返回false + */ + public boolean getFromDB(Connection conn) { + try { + List stables = getSTables(conn); + if (!stables.contains(stable)) { + LOG.error("超级表{}不存在，无法从数据库获取表结构信息.", stable); + return false; + } + } catch (SQLException e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + return false; + } + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("describe " + stable); + int colIndex = 0; + while (rs.next()) { + String name = rs.getString(1); + String type = rs.getString(2); + String note = rs.getString(4); + if ("TIMESTAMP".equals(type)) { + tsColName = name; + tsColIndex = colIndex; + } else if ("TAG".equals(note)) { + tagIndexMap.put(name, colIndex); + tagList.add(name); + } else { + fieldIndexMap.put(name, colIndex); + fieldList.add(name); + } + colIndex++; + } + LOG.info("从数据库获取的表结构概要：tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); + return true; + } catch (SQLException e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + return false; + } + } + + public static List getSTables(Connection conn) throws SQLException { + List stables = new ArrayList<>(); + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("show stables"); + while (rs.next()) { + String name = rs.getString(1); + stables.add(name); + } + } + return stables; + } + + public void createSTable(Connection conn, Record record) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE STABLE IF NOT EXISTS ").append(stable).append("("); + sb.append(tsColName).append(" ").append("TIMESTAMP,"); + for (String fieldName : fieldList) { + sb.append(fieldName).append(' '); + Column col = record.getColumn(fieldIndexMap.get(fieldName)); + String tdType = mapDataxType(col.getType()); + sb.append(tdType).append(','); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(") TAGS("); + for (String tagName : tagList) { + sb.append(tagName).append(" NCHAR(64),"); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + String q = sb.toString(); + LOG.info("自动创建超级表：" + q); + try (Statement stmt = conn.createStatement()) { + stmt.execute(q); + } + } + + public String[] getTagValuesFromRecord(Record record) { + String[] tagValues = new String[tagList.size()]; + for (int i = 0; i < tagList.size(); ++i) { + if (fixedTagValue.containsKey(tagList.get(i))) { + tagValues[i] = fixedTagValue.get(tagList.get(i)); + } else { + int tagIndex = tagIndexMap.get(tagList.get(i)); + tagValues[i] = record.getColumn(tagIndex).asString(); + } + } + return tagValues; + } + + public Map getFieldIndexMap() { + return fieldIndexMap; + } + + public List getFieldList() { + return fieldList; + } + + public String getJoinedFieldNames() { + return tsColName + ", " + String.join(", ", fieldList); + } + + public int getTsColIndex() { + return tsColIndex; + } + + public String getTagValuesPlaceHolder() { + return tagList.stream().map(x -> "?").collect(Collectors.joining(",")); + } + + public String getFieldValuesPlaceHolder() { + return "?, " + fieldList.stream().map(x -> "?").collect(Collectors.joining(", ")); + } + + /** + * 计算子表表名 + *
    + *
  1. 将标签的value 组合成为如下的字符串: tag_value1!tag_value2!tag_value3。
    2. + *
  2. 计算该字符串的 MD5 散列值 "md5_val"。
    3. + *
  3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。
    4. + *
+ * + * @param tagValues + * @return + */ + public String computeTableName(String[] tagValues) { + String s = String.join("!", tagValues); + return "t_" + DigestUtils.md5Hex(s); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java similarity index 84% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java index 8460080..70ea573 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.plugin.RecordReceiver; @@ -64,7 +64,13 @@ public class TDengineWriter extends Writer { String value = this.writerSliceConfig.getString(key); properties.setProperty(key, value); } - + if (!keys.contains(Key.USER)) { + properties.setProperty(Key.USER, "root"); + } + if (!keys.contains(Key.PASSWORD)) { + properties.setProperty(Key.PASSWORD, "taosdata"); + } + LOG.debug("========================properties==========================\n" + properties.toString()); String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); LOG.debug("start to handle record from: " + peerPluginName); DataHandler handler = DataHandlerFactory.build(peerPluginName); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java similarity index 75% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java index 02e8707..994f1e8 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/TDengineWriterErrorCode.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java @@ -1,9 +1,10 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.spi.ErrorCode; public enum TDengineWriterErrorCode implements ErrorCode { - RUNTIME_EXCEPTION("TDengineWriter-00", "运行时异常"); + RUNTIME_EXCEPTION("TDengineWriter-00", "运行时异常"), + TYPE_ERROR("TDengineWriter-00", "Datax类型无法正确映射到TDengine类型"); private final String code; private final String description; diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h similarity index 100% rename from tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/com_alibaba_datax_plugin_writer_JniConnection.h rename to tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h diff --git a/tdenginewriter/src/main/resources/plugin.json b/tdenginewriter/src/main/resources/plugin.json index 6c900a1..e54f65f 100755 --- a/tdenginewriter/src/main/resources/plugin.json +++ b/tdenginewriter/src/main/resources/plugin.json @@ -1,9 +1,9 @@ { "name": "tdenginewriter", - "class": "com.alibaba.datax.plugin.writer.TDengineWriter", + "class": "com.alibaba.datax.plugin.writer.tdenginewriter.TDengineWriter", "description": { "useScene": "data migration to tdengine", - "mechanism": "use JNI to write data to tdengine." + "mechanism": "use JNI or taos-jdbc to write data to tdengine." }, "developer": "zyyang-taosdata" } \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java similarity index 90% rename from tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java rename to tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java index 040cf34..09c3df2 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/JniConnectionTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java @@ -1,4 +1,4 @@ -package com.alibaba.datax.plugin.writer; +package com.alibaba.datax.plugin.writer.tdenginewriter; import org.junit.Test; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java new file mode 100644 index 0000000..43928db --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java @@ -0,0 +1,21 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class TDengineWriterTest { + + + @Test + public void testGetSchema() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", "wozai.fun", "6030", "test", "root", "taosdata"); + Connection conn = DriverManager.getConnection(jdbcUrl); + SchemaManager schemaManager = new SchemaManager(); + schemaManager.setStable("test1"); + schemaManager.getFromDB(conn); + } +} From ca1851fb995a3200bd4c186539ff1930b7162e8d Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 18 Nov 2021 18:06:10 +0800 Subject: [PATCH 035/114] mongodb2tdengine test and refine --- core/src/main/job/mongodb2tdengine.json | 4 +- tdenginewriter/doc/tdenginewriter.md | 11 +- .../writer/tdenginewriter/DataHandler.java | 3 +- .../tdenginewriter/DefaultDataHandler.java | 17 +-- .../tdenginewriter/JDBCBatchWriter.java | 100 ++++++++++++++++-- .../tdenginewriter/OpentsdbDataHandler.java | 3 +- .../writer/tdenginewriter/SchemaManager.java | 25 ++++- .../writer/tdenginewriter/TDengineWriter.java | 2 +- .../tdenginewriter/TDengineWriterTest.java | 10 ++ 9 files changed, 150 insertions(+), 25 deletions(-) diff --git a/core/src/main/job/mongodb2tdengine.json b/core/src/main/job/mongodb2tdengine.json index 4cfc987..45e5a64 100644 --- a/core/src/main/job/mongodb2tdengine.json +++ b/core/src/main/job/mongodb2tdengine.json @@ -11,7 +11,7 @@ "name": "mongodbreader", "parameter": { "address": [ - "123.56.104.14:27017" + "127.0.0.1:27017" ], "userName": "admin678", "mechanism": "SCRAM-SHA-1", @@ -50,7 +50,7 @@ "writer": { "name": "tdenginewriter", "parameter": { - "host": "123.56.104.14", + "host": "127.0.0.1", "port": 6030, "dbname": "test", "user": "root", diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index c9c222a..9ab64a2 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -290,6 +290,9 @@ TAGS( ## 5 约束限制 +1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 +2. 标签列不能包含null值，如果包含会被过滤掉。 + ## FAQ ### 如何选取要同步的数据的范围？ @@ -300,10 +303,14 @@ TAGS( 如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 -### 1张源表导入之后对应TDengine中多少张表？ +### 一张源表导入之后对应TDengine中多少张表？ 这是又tagColumn决定的，如果所有tag列的值都相同，目标表也只有一个。源表有多少不同的tag组合，目标超表就会有多少子表。 ### 源表和目标表的字段顺序一致吗？ -TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表是自动调整。 \ No newline at end of file +TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表是自动调整。 + +### 插件如何确定各列的数据类型？ + +抽样收到的第一批数据自动推断各列的类型。schema是从数据来的，因此要保障“好的”数据占大多数。 \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java index 686ac27..421c2fe 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java @@ -1,10 +1,11 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; import java.util.Properties; public interface DataHandler { - long handle(RecordReceiver lineReceiver, Properties properties); + long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector); } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 733f49c..9250910 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -2,6 +2,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; import com.taosdata.jdbc.TSDBPreparedStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +17,6 @@ import java.util.Properties; */ public class DefaultDataHandler implements DataHandler { private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); - static { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); @@ -26,7 +26,7 @@ public class DefaultDataHandler implements DataHandler { } @Override - public long handle(RecordReceiver lineReceiver, Properties properties) { + public long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector) { SchemaManager schemaManager = new SchemaManager(properties); if (!schemaManager.configValid()) { return 0; @@ -47,7 +47,11 @@ public class DefaultDataHandler implements DataHandler { } int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, "1000")); - return write(lineReceiver, conn, batchSize, schemaManager); + if (batchSize < 5) { + LOG.error("batchSize太小，会增加自动类型推断错误的概率，建议改大后重试"); + return 0; + } + return write(lineReceiver, conn, batchSize, schemaManager, collector); } catch (Exception e) { LOG.error("write failed " + e.getMessage()); e.printStackTrace(); @@ -79,18 +83,15 @@ public class DefaultDataHandler implements DataHandler { * @return 成功写入记录数 * @throws SQLException */ - private long write(RecordReceiver lineReceiver, Connection conn, int batchSize, SchemaManager scm) throws SQLException { + private long write(RecordReceiver lineReceiver, Connection conn, int batchSize, SchemaManager scm, TaskPluginCollector collector) throws SQLException { Record record = lineReceiver.getFromReader(); if (record == null) { return 0; } - if (scm.shouldCreateTable()) { - scm.createSTable(conn, record); - } String pq = String.format("INSERT INTO ? USING %s TAGS(%s) (%s) values (%s)", scm.getStable(), scm.getTagValuesPlaceHolder(), scm.getJoinedFieldNames(), scm.getFieldValuesPlaceHolder()); LOG.info("Prepared SQL: {}", pq); try (TSDBPreparedStatement stmt = (TSDBPreparedStatement) conn.prepareStatement(pq)) { - JDBCBatchWriter batchWriter = new JDBCBatchWriter(stmt, scm, batchSize); + JDBCBatchWriter batchWriter = new JDBCBatchWriter(conn, stmt, scm, batchSize, collector); do { batchWriter.append(record); } while ((record = lineReceiver.getFromReader()) != null); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java index 17023d0..21974e9 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -3,10 +3,13 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.AbstractTaskPlugin; +import com.alibaba.datax.common.plugin.TaskPluginCollector; import com.taosdata.jdbc.TSDBPreparedStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; @@ -26,10 +29,12 @@ import java.util.stream.Collectors; */ public class JDBCBatchWriter { public static final Logger LOG = LoggerFactory.getLogger(JDBCBatchWriter.class); - private TSDBPreparedStatement stmt; private SchemaManager scm; + private Connection conn; private int batchSize; + private TaskPluginCollector collector; + // 缓存Record， key为tableName Map> buf = new HashMap<>(); // 缓存表的标签值， key为tableName @@ -37,25 +42,57 @@ public class JDBCBatchWriter { private long sucCount = 0; private final int tsColIndex; private List fieldList; + // 每个record至少应该包含的列数，用于检测数据 + private int minColNum = 0; private Map fieldIndexMap; + private List fieldTypes = null; - public JDBCBatchWriter(TSDBPreparedStatement stmt, SchemaManager scm, int batchSize) { + public JDBCBatchWriter(Connection conn, TSDBPreparedStatement stmt, SchemaManager scm, int batchSize, TaskPluginCollector collector) { + this.conn = conn; this.stmt = stmt; this.scm = scm; this.batchSize = batchSize; + this.collector = collector; this.tsColIndex = scm.getTsColIndex(); this.fieldList = scm.getFieldList(); this.fieldIndexMap = scm.getFieldIndexMap(); + this.minColNum = 1 + fieldList.size() + scm.getDynamicTagCount(); + } + public void initFiledTypesAndTargetTable(List records) throws SQLException { + if (fieldTypes != null) { + return; + } + guessFieldTypes(records); + if (scm.shouldCreateTable()) { + scm.createSTable(conn, fieldTypes); + } + } public void append(Record record) throws SQLException { + int columnNum = record.getColumnNumber(); + if (columnNum < minColNum) { + collector.collectDirtyRecord(record, "实际列数小于期望列数"); + return; + } String[] tagValues = scm.getTagValuesFromRecord(record); + if (tagValues == null) { + collector.collectDirtyRecord(record, "标签列包含null"); + return; + } + if (!scm.hasTimestamp(record)) { + collector.collectDirtyRecord(record, "时间戳列为null或类型错误"); + return; + } String tableName = scm.computeTableName(tagValues); if (buf.containsKey(tableName)) { List lis = buf.get(tableName); lis.add(record); if (lis.size() == batchSize) { + if (fieldTypes == null) { + initFiledTypesAndTargetTable(lis); + } executeBatch(tableName); lis.clear(); } @@ -67,6 +104,49 @@ public class JDBCBatchWriter { } } + /** + * 只有String类型比较特别，测试发现值为null的列会转成String类型。所以Column的类型为String并不代表这一列的类型真的是String。 + * + * @param records + */ + private void guessFieldTypes(List records) { + fieldTypes = new ArrayList<>(fieldList.size()); + for (int i = 0; i < fieldList.size(); ++i) { + int colIndex = fieldIndexMap.get(fieldList.get(i)); + boolean ok = false; + for (int j = 0; j < records.size() && !ok; ++j) { + Column column = records.get(j).getColumn(colIndex); + Column.Type type = column.getType(); + switch (type) { + case LONG: + case DOUBLE: + case DATE: + case BOOL: + case BYTES: + if (column.getRawData() != null) { + fieldTypes.add(type); + ok = true; + } + break; + case STRING: + // 只有非null且非空的String列，才会被真的当作String类型。 + String value = column.asString(); + if (value != null && !"".equals(value)) { + fieldTypes.add(type); + ok = true; + } + break; + default: + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, fieldTypes.get(i).toString()); + } + } + if (!ok) { + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, String.format("根据采样的%d条数据，无法推断第%d列的数据类型", records.size(), i + 1)); + } + } + LOG.info("Field Types: {}", fieldTypes); + } + /** * 执行单表批量写入 * @@ -87,12 +167,10 @@ public class JDBCBatchWriter { ArrayList tsList = records.stream().map(r -> r.getColumn(tsColIndex).asDate().getTime()).collect(Collectors.toCollection(ArrayList::new)); stmt.setTimestamp(0, tsList); // 字段 - Record record = records.get(0); for (int i = 0; i < fieldList.size(); ) { String fieldName = fieldList.get(i); int index = fieldIndexMap.get(fieldName); - Column column = record.getColumn(index); - switch (column.getType()) { + switch (fieldTypes.get(i)) { case LONG: ArrayList lisLong = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); stmt.setLong(++i, lisLong); @@ -118,7 +196,7 @@ public class JDBCBatchWriter { stmt.setString(++i, lisBytes, 64); break; default: - throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, column.getType().toString()); + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, fieldTypes.get(i).toString()); } } // 执行 @@ -132,6 +210,16 @@ public class JDBCBatchWriter { * 把缓存的Record全部写入 */ public void flush() throws SQLException { + if (fieldTypes == null) { + List records = new ArrayList<>(); + for (List lis : buf.values()) { + records.addAll(lis); + if (records.size() > 100) { + break; + } + } + initFiledTypesAndTargetTable(records); + } for (String tabName : buf.keySet()) { if (buf.get(tabName).size() > 0) { executeBatch(tabName); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java index 52f1aa7..e1b8f5d 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java @@ -4,6 +4,7 @@ import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,7 +15,7 @@ public class OpentsdbDataHandler implements DataHandler { private static final String DEFAULT_BATCH_SIZE = "1"; @Override - public long handle(RecordReceiver lineReceiver, Properties properties) { + public long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector) { // opentsdb json protocol use JNI and schemaless API to write String host = properties.getProperty(Key.HOST); int port = Integer.parseInt(properties.getProperty(Key.PORT)); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index b3d7b7e..21b8ef0 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -176,14 +176,15 @@ public class SchemaManager { return stables; } - public void createSTable(Connection conn, Record record) throws SQLException { + public void createSTable(Connection conn, List fieldTypes) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("CREATE STABLE IF NOT EXISTS ").append(stable).append("("); sb.append(tsColName).append(" ").append("TIMESTAMP,"); - for (String fieldName : fieldList) { + for (int i = 0; i < fieldList.size(); ++i) { + String fieldName = fieldList.get(i); + Column.Type dxType = fieldTypes.get(i); sb.append(fieldName).append(' '); - Column col = record.getColumn(fieldIndexMap.get(fieldName)); - String tdType = mapDataxType(col.getType()); + String tdType = mapDataxType(dxType); sb.append(tdType).append(','); } sb.deleteCharAt(sb.length() - 1); @@ -209,10 +210,22 @@ public class SchemaManager { int tagIndex = tagIndexMap.get(tagList.get(i)); tagValues[i] = record.getColumn(tagIndex).asString(); } + if (tagValues[i] == null) { + return null; + } } return tagValues; } + public boolean hasTimestamp(Record record) { + Column column = record.getColumn(tsColIndex); + if (column.getType() == Column.Type.DATE && column.asDate() != null) { + return true; + } else { + return false; + } + } + public Map getFieldIndexMap() { return fieldIndexMap; } @@ -252,4 +265,8 @@ public class SchemaManager { String s = String.join("!", tagValues); return "t_" + DigestUtils.md5Hex(s); } + + public int getDynamicTagCount() { + return tagIndexMap.size(); + } } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java index 70ea573..cd22379 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -74,7 +74,7 @@ public class TDengineWriter extends Writer { String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); LOG.debug("start to handle record from: " + peerPluginName); DataHandler handler = DataHandlerFactory.build(peerPluginName); - long records = handler.handle(lineReceiver, properties); + long records = handler.handle(lineReceiver, properties, getTaskPluginCollector()); LOG.debug("handle data finished, records: " + records); } diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java index 43928db..62bf704 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java @@ -5,6 +5,7 @@ import org.junit.Test; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; public class TDengineWriterTest { @@ -18,4 +19,13 @@ public class TDengineWriterTest { schemaManager.setStable("test1"); schemaManager.getFromDB(conn); } + + @Test + public void dropTestTable() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", "wozai.fun", "6030", "test", "root", "taosdata"); + Connection conn = DriverManager.getConnection(jdbcUrl); + Statement stmt = conn.createStatement(); + stmt.execute("drop table market_snapshot"); + } } From 75d4f7e101bc17bf0427acc7efcdafb4648642ff Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 18 Nov 2021 19:08:19 +0800 Subject: [PATCH 036/114] mongodb2tdengine check records count before flush --- .../datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java index 21974e9..3b1f860 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -218,7 +218,11 @@ public class JDBCBatchWriter { break; } } - initFiledTypesAndTargetTable(records); + if (records.size() > 0) { + initFiledTypesAndTargetTable(records); + } else { + return; + } } for (String tabName : buf.keySet()) { if (buf.get(tabName).size() > 0) { From 485d2d881593aaadc0ad39c95aa215e88d2f38db Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 18 Nov 2021 20:18:40 +0800 Subject: [PATCH 037/114] mongodb2tdengine typo --- tdenginewriter/doc/tdenginewriter.md | 2 +- .../datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java | 1 - .../datax/plugin/writer/tdenginewriter/SchemaManager.java | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 9ab64a2..715080c 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -197,7 +197,7 @@ CREATE STABLE IF NOT EXISTS market_snapshot ( ) TAGS( industry NCHAR(64), - stockID NCHAR(64 + stockID NCHAR(64) ); ``` diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java index 3b1f860..279e6ed 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -3,7 +3,6 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.exception.DataXException; -import com.alibaba.datax.common.plugin.AbstractTaskPlugin; import com.alibaba.datax.common.plugin.TaskPluginCollector; import com.taosdata.jdbc.TSDBPreparedStatement; import org.slf4j.Logger; diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index 21b8ef0..22c8a44 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -48,7 +48,7 @@ public class SchemaManager { case BOOL: return "BOOL"; case BYTES: - return "BINARY"; + return "BINARY(64)"; default: throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, type.toString()); } From be78295e116f6b2c98e65bb76c3d5cf2b71f2578 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 18 Nov 2021 22:57:53 +0800 Subject: [PATCH 038/114] mongodb2tdengine typo --- tdenginewriter/doc/tdenginewriter.md | 4 ++-- .../datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 715080c..432b1fb 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -305,7 +305,7 @@ TAGS( ### 一张源表导入之后对应TDengine中多少张表？ -这是又tagColumn决定的，如果所有tag列的值都相同，目标表也只有一个。源表有多少不同的tag组合，目标超表就会有多少子表。 +这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超表就有多少子表。 ### 源表和目标表的字段顺序一致吗？ @@ -313,4 +313,4 @@ TDengine要求每个表第一列是时间戳列，后边是普通字段，最后 ### 插件如何确定各列的数据类型？ -抽样收到的第一批数据自动推断各列的类型。schema是从数据来的，因此要保障“好的”数据占大多数。 \ No newline at end of file +根据收到的第一批数据自动推断各列的类型。 \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java index 279e6ed..20065a7 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -41,7 +41,7 @@ public class JDBCBatchWriter { private long sucCount = 0; private final int tsColIndex; private List fieldList; - // 每个record至少应该包含的列数，用于检测数据 + // 每个record至少应该包含的列数，用于校验数据 private int minColNum = 0; private Map fieldIndexMap; private List fieldTypes = null; From e5c3fed1a939e8006911a423f7559ff5b7428a1c Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 19 Nov 2021 14:20:37 +0800 Subject: [PATCH 039/114] i18n support for some import log messages --- .../tdenginewriter/DefaultDataHandler.java | 7 ++++-- .../tdenginewriter/JDBCBatchWriter.java | 12 ++++++--- .../plugin/writer/tdenginewriter/Msg.java | 20 +++++++++++++++ .../writer/tdenginewriter/SchemaManager.java | 13 +++++----- .../resources/tdenginewritermsg.properties | 6 +++++ .../tdenginewritermsg_en_US.properties | 6 +++++ .../tdenginewritermsg_zh_CN.properties | 6 +++++ .../writer/tdenginewriter/MessageTest.java | 25 +++++++++++++++++++ 8 files changed, 82 insertions(+), 13 deletions(-) create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java create mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg.properties create mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties create mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 9250910..91c2b7e 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -17,6 +17,7 @@ import java.util.Properties; */ public class DefaultDataHandler implements DataHandler { private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); + static { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); @@ -38,7 +39,8 @@ public class DefaultDataHandler implements DataHandler { return 0; } if (schemaManager.shouldGuessSchema()) { - LOG.info("无法从配置文件获取表结构信息，尝试从数据库获取"); + // 无法从配置文件获取表结构信息，尝试从数据库获取 + LOG.info(Msg.get("try_get_schema_from_db")); boolean success = schemaManager.getFromDB(conn); if (!success) { return 0; @@ -48,7 +50,8 @@ public class DefaultDataHandler implements DataHandler { } int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, "1000")); if (batchSize < 5) { - LOG.error("batchSize太小，会增加自动类型推断错误的概率，建议改大后重试"); + // batchSize太小，会增加自动类型推断错误的概率，建议改大后重试 + LOG.error(Msg.get("batch_size_too_small")); return 0; } return write(lineReceiver, conn, batchSize, schemaManager, collector); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java index 20065a7..53ab9bb 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -72,16 +72,19 @@ public class JDBCBatchWriter { public void append(Record record) throws SQLException { int columnNum = record.getColumnNumber(); if (columnNum < minColNum) { - collector.collectDirtyRecord(record, "实际列数小于期望列数"); + // 实际列数小于期望列数 + collector.collectDirtyRecord(record, Msg.get("column_number_error")); return; } String[] tagValues = scm.getTagValuesFromRecord(record); if (tagValues == null) { - collector.collectDirtyRecord(record, "标签列包含null"); + // 标签列包含null + collector.collectDirtyRecord(record, Msg.get("tag_value_error")); return; } if (!scm.hasTimestamp(record)) { - collector.collectDirtyRecord(record, "时间戳列为null或类型错误"); + // 时间戳列为null或类型错误 + collector.collectDirtyRecord(record, Msg.get("ts_value_error")); return; } String tableName = scm.computeTableName(tagValues); @@ -140,7 +143,8 @@ public class JDBCBatchWriter { } } if (!ok) { - throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, String.format("根据采样的%d条数据，无法推断第%d列的数据类型", records.size(), i + 1)); + // 根据采样的%d条数据，无法推断第%d列的数据类型 + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, String.format(Msg.get("infer_column_type_error"), records.size(), i + 1)); } } LOG.info("Field Types: {}", fieldTypes); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java new file mode 100644 index 0000000..89730d3 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java @@ -0,0 +1,20 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import java.util.Locale; +import java.util.ResourceBundle; + +/** + * i18n message util + */ +public class Msg { + private static ResourceBundle bundle; + + static { + bundle = ResourceBundle.getBundle("tdenginewritermsg", Locale.getDefault()); + } + + public static String get(String key) { + return bundle.getString(key); + } + +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index 22c8a44..d67a658 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -65,7 +65,7 @@ public class SchemaManager { private void getFromConfig(Properties properties) { stable = properties.getProperty(Key.STABLE); if (stable == null) { - LOG.error("配置错误: no stable"); + LOG.error("Config error: no stable"); return; } for (Object key : properties.keySet()) { @@ -98,7 +98,7 @@ public class SchemaManager { List sortedFieldName = fieldIndexMap.entrySet().stream().sorted((x, y) -> x.getValue().compareTo(y.getValue())).map(e -> e.getKey()).collect(Collectors.toList()); fieldList.addAll(sortedFieldName); // 排序的目的是保证自动建表时列的顺序和输入数据的列的顺序保持一致 canInferSchemaFromConfig = tsColIndex > -1 && !(fixedTagValue.isEmpty() && tagIndexMap.isEmpty()) && !fieldIndexMap.isEmpty(); - LOG.info("配置文件解析结果：fixedTags=[{}] ,tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", fixedTagValue.keySet()), String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); + LOG.info("Config file parsed result：fixedTags=[{}] ,tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", fixedTagValue.keySet()), String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); } public boolean shouldGuessSchema() { @@ -112,8 +112,7 @@ public class SchemaManager { public boolean configValid() { boolean valid = (tagList.size() > 0 && fieldList.size() > 0 && tsColIndex > -1) || (tagList.size() == 0 && fieldList.size() == 0 && tsColIndex == -1); if (!valid) { - LOG.error("配置错误. tag_columns，field_columns，timestamp_column必须同时存在或同时省略，当前解析结果: tag_columns: {}, field_columns:{}, timestamp_column:{} tsColIndex:{}", - (fixedTagValue.size() + tagIndexMap.size()), fieldIndexMap.size(), tsColName, tsColIndex); + LOG.error("Config error: tagColumn, fieldColumn and timestampColumn must be present together or absent together."); } return valid; } @@ -128,7 +127,7 @@ public class SchemaManager { try { List stables = getSTables(conn); if (!stables.contains(stable)) { - LOG.error("超级表{}不存在，无法从数据库获取表结构信息.", stable); + LOG.error("super table {} not exist， fail to get schema from database.", stable); return false; } } catch (SQLException e) { @@ -155,7 +154,7 @@ public class SchemaManager { } colIndex++; } - LOG.info("从数据库获取的表结构概要：tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); + LOG.info("table info：tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); return true; } catch (SQLException e) { LOG.error(e.getMessage()); @@ -195,7 +194,7 @@ public class SchemaManager { sb.deleteCharAt(sb.length() - 1); sb.append(")"); String q = sb.toString(); - LOG.info("自动创建超级表：" + q); + LOG.info("run sql：" + q); try (Statement stmt = conn.createStatement()) { stmt.execute(q); } diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg.properties b/tdenginewriter/src/main/resources/tdenginewritermsg.properties new file mode 100644 index 0000000..4aaa220 --- /dev/null +++ b/tdenginewriter/src/main/resources/tdenginewritermsg.properties @@ -0,0 +1,6 @@ +try_get_schema_fromdb=fail to get structure info of target table from configure file and will try to get it from database +batch_size_too_small='batchSize' is too small, please increase it and try again +column_number_error=number of columns is less than expected +tag_value_error=tag columns include 'null' value +ts_value_error=timestamp column type error or null +infer_column_type_error=fail to infer column type: sample count %d, column index %d \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties new file mode 100644 index 0000000..4aaa220 --- /dev/null +++ b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties @@ -0,0 +1,6 @@ +try_get_schema_fromdb=fail to get structure info of target table from configure file and will try to get it from database +batch_size_too_small='batchSize' is too small, please increase it and try again +column_number_error=number of columns is less than expected +tag_value_error=tag columns include 'null' value +ts_value_error=timestamp column type error or null +infer_column_type_error=fail to infer column type: sample count %d, column index %d \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties new file mode 100644 index 0000000..4b9552f --- /dev/null +++ b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties @@ -0,0 +1,6 @@ +try_get_schema_fromdb=\u65e0\u6cd5\u4ece\u914d\u7f6e\u6587\u4ef6\u83b7\u53d6\u8868\u7ed3\u6784\u4fe1\u606f\uff0c\u5c1d\u8bd5\u4ece\u6570\u636e\u5e93\u83b7\u53d6 +batch_size_too_small=batchSize\u592a\u5c0f\uff0c\u4f1a\u589e\u52a0\u81ea\u52a8\u7c7b\u578b\u63a8\u65ad\u9519\u8bef\u7684\u6982\u7387\uff0c\u5efa\u8bae\u6539\u5927\u540e\u91cd\u8bd5 +column_number_error=\u5b9e\u9645\u5217\u6570\u5c0f\u4e8e\u671f\u671b\u5217\u6570 +tag_value_error=\u6807\u7b7e\u5217\u5305\u542bnull +ts_value_error=\u65f6\u95f4\u6233\u5217\u4e3anull\u6216\u7c7b\u578b\u9519\u8bef +infer_column_type_error=\u6839\u636e\u91c7\u6837\u7684%d\u6761\u6570\u636e\uff0c\u65e0\u6cd5\u63a8\u65ad\u7b2c%d\u5217\u7684\u6570\u636e\u7c7b\u578b diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java new file mode 100644 index 0000000..b1b7ddd --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java @@ -0,0 +1,25 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import org.junit.Test; + +import java.util.Locale; +import java.util.ResourceBundle; + +import org.junit.Assert; + +public class MessageTest { + @Test + public void testChineseMessage() { + Locale local = new Locale("zh", "CN"); + ResourceBundle bundle = ResourceBundle.getBundle("tdenginewritermsg", local); + String msg = bundle.getString("try_get_schema_fromdb"); + Assert.assertEquals("无法从配置文件获取表结构信息，尝试从数据库获取", msg); + } + + @Test + public void testDefaultMessage() { + ResourceBundle bundle = ResourceBundle.getBundle("tdenginewritermsg", Locale.getDefault()); + String msg = bundle.getString("try_get_schema_fromdb"); + System.out.println(msg); + } +} From b47cdaf217dafa3ff84b8a289b13fc5e190901b5 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 19 Nov 2021 14:23:24 +0800 Subject: [PATCH 040/114] use plain user name in demo config file --- core/src/main/job/mongodb2tdengine.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/job/mongodb2tdengine.json b/core/src/main/job/mongodb2tdengine.json index 45e5a64..49e04c1 100644 --- a/core/src/main/job/mongodb2tdengine.json +++ b/core/src/main/job/mongodb2tdengine.json @@ -13,9 +13,9 @@ "address": [ "127.0.0.1:27017" ], - "userName": "admin678", + "userName": "mongouser", "mechanism": "SCRAM-SHA-1", - "userPassword": "huwG86123", + "userPassword": "mongopass", "authDb": "admin", "dbName": "test", "collectionName": "cu_market_data", From 17c39b11bac472f6a0bac1495a92aed45c602766 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 19 Nov 2021 16:38:55 +0800 Subject: [PATCH 041/114] migrate datax.py to python3, and compatible with python2 --- core/src/main/bin/datax.py | 114 +++++++++++++++++++++---------------- userGuid.md | 2 +- 2 files changed, 65 insertions(+), 51 deletions(-) diff --git a/core/src/main/bin/datax.py b/core/src/main/bin/datax.py index 1099ed3..4811ae8 100755 --- a/core/src/main/bin/datax.py +++ b/core/src/main/bin/datax.py @@ -1,23 +1,26 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- -import sys -import os -import signal -import subprocess -import time -import re -import socket -import json -from optparse import OptionParser -from optparse import OptionGroup -from string import Template import codecs +import json +import os import platform +import re +import signal +import socket +import subprocess +import sys +import time +from optparse import OptionGroup +from optparse import OptionParser +from string import Template + +ispy2 = sys.version_info.major == 2 def isWindows(): return platform.system() == 'Windows' + DATAX_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATAX_VERSION = 'DATAX-OPENSOURCE-3.0' @@ -52,13 +55,19 @@ def getLocalIp(): def suicide(signum, e): global child_process - print >> sys.stderr, "[Error] DataX receive unexpected signal %d, starts to suicide." % (signum) + if ispy2: + print >> sys.stderr, "[Error] DataX receive unexpected signal %d, starts to suicide." % (signum) + else: + print("[Error] DataX receive unexpected signal %d, starts to suicide." % (signum), sys.stderr) if child_process: child_process.send_signal(signal.SIGQUIT) time.sleep(1) child_process.kill() - print >> sys.stderr, "DataX Process was killed ! you did ?" + if ispy2: + print >> sys.stderr, "DataX Process was killed ! you did ?" + else: + print("DataX Process was killed ! you did ?", sys.stderr) sys.exit(RET_STATE["KILL"]) @@ -92,10 +101,10 @@ def getOptionParser(): 'if you have mutiple parameters: -p"-DtableName=your-table-name -DcolumnName=your-column-name".' 'Note: you should config in you job tableName with ${tableName}.') prodEnvOptionGroup.add_option("-r", "--reader", metavar="", - action="store", dest="reader",type="string", + action="store", dest="reader", type="string", help='View job config[reader] template, eg: mysqlreader,streamreader') prodEnvOptionGroup.add_option("-w", "--writer", metavar="", - action="store", dest="writer",type="string", + action="store", dest="writer", type="string", help='View job config[writer] template, eg: mysqlwriter,streamwriter') parser.add_option_group(prodEnvOptionGroup) @@ -108,45 +117,50 @@ def getOptionParser(): parser.add_option_group(devEnvOptionGroup) return parser + def generateJobConfigTemplate(reader, writer): - readerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n" % (reader,reader,reader) - writerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n " % (writer,writer,writer) - print readerRef - print writerRef + readerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n" % ( + reader, reader, reader) + writerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n " % ( + writer, writer, writer) + print(readerRef) + print(writerRef) jobGuid = 'Please save the following configuration as a json file and use\n python {DATAX_HOME}/bin/datax.py {JSON_FILE_NAME}.json \nto run the job.\n' - print jobGuid - jobTemplate={ - "job": { - "setting": { - "speed": { - "channel": "" - } - }, - "content": [ - { - "reader": {}, - "writer": {} - } - ] - } + print(jobGuid) + jobTemplate = { + "job": { + "setting": { + "speed": { + "channel": "" + } + }, + "content": [ + { + "reader": {}, + "writer": {} + } + ] + } } - readerTemplatePath = "%s/plugin/reader/%s/plugin_job_template.json" % (DATAX_HOME,reader) - writerTemplatePath = "%s/plugin/writer/%s/plugin_job_template.json" % (DATAX_HOME,writer) + readerTemplatePath = "%s/plugin/reader/%s/plugin_job_template.json" % (DATAX_HOME, reader) + writerTemplatePath = "%s/plugin/writer/%s/plugin_job_template.json" % (DATAX_HOME, writer) try: - readerPar = readPluginTemplate(readerTemplatePath); - except Exception, e: - print "Read reader[%s] template error: can\'t find file %s" % (reader,readerTemplatePath) + readerPar = readPluginTemplate(readerTemplatePath) + except: + print("Read reader[%s] template error: can\'t find file %s" % (reader, readerTemplatePath)) try: - writerPar = readPluginTemplate(writerTemplatePath); - except Exception, e: - print "Read writer[%s] template error: : can\'t find file %s" % (writer,writerTemplatePath) - jobTemplate['job']['content'][0]['reader'] = readerPar; - jobTemplate['job']['content'][0]['writer'] = writerPar; - print json.dumps(jobTemplate, indent=4, sort_keys=True) + writerPar = readPluginTemplate(writerTemplatePath) + except: + print("Read writer[%s] template error: : can\'t find file %s" % (writer, writerTemplatePath)) + jobTemplate['job']['content'][0]['reader'] = readerPar + jobTemplate['job']['content'][0]['writer'] = writerPar + print(json.dumps(jobTemplate, indent=4, sort_keys=True)) + def readPluginTemplate(plugin): with open(plugin, 'r') as f: - return json.load(f) + return json.load(f) + def isUrl(path): if not path: @@ -168,7 +182,7 @@ def buildStartCommand(options, args): if options.remoteDebug: tempJVMCommand = tempJVMCommand + " " + REMOTE_DEBUG_CONFIG - print 'local ip: ', getLocalIp() + print('local ip: ', getLocalIp()) if options.loglevel: tempJVMCommand = tempJVMCommand + " " + ("-Dloglevel=%s" % (options.loglevel)) @@ -198,11 +212,11 @@ def buildStartCommand(options, args): def printCopyright(): - print ''' + print(''' DataX (%s), From Alibaba ! Copyright (C) 2010-2017, Alibaba Group. All Rights Reserved. -''' % DATAX_VERSION +''' % DATAX_VERSION) sys.stdout.flush() @@ -211,7 +225,7 @@ if __name__ == "__main__": parser = getOptionParser() options, args = parser.parse_args(sys.argv[1:]) if options.reader is not None and options.writer is not None: - generateJobConfigTemplate(options.reader,options.writer) + generateJobConfigTemplate(options.reader, options.writer) sys.exit(RET_STATE['OK']) if len(args) != 1: parser.print_help() diff --git a/userGuid.md b/userGuid.md index 153c811..16771a5 100644 --- a/userGuid.md +++ b/userGuid.md @@ -10,7 +10,7 @@ DataX本身作为数据同步框架，将不同数据源的同步抽象为从源 - Linux - [JDK(1.8以上，推荐1.8) ](http://www.oracle.com/technetwork/cn/java/javase/downloads/index.html) -- [Python(推荐Python2.6.X) ](https://www.python.org/downloads/) +- [Python(2或3都可以) ](https://www.python.org/downloads/) - [Apache Maven 3.x](https://maven.apache.org/download.cgi) (Compile DataX) # Quick Start From 24952af5c3a3b9cfa5be4ac5b3bb2860bab6a868 Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 23 Nov 2021 08:56:57 +0800 Subject: [PATCH 042/114] create test case --- .../datax/core/TestMysql2TDengine.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java diff --git a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java new file mode 100644 index 0000000..aeb0507 --- /dev/null +++ b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java @@ -0,0 +1,20 @@ +package com.alibaba.datax.core; + + +import org.junit.Test; + +public class TestMysql2TDengine { + + @Test + public void test() { + System.out.println(System.getProperty("java.library.path")); + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/main/job/mysql2tdengine.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + try { + Engine.entry(params); + } catch (Throwable e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file From 93a3369eb6346607ad1fdca7055f6e72645b96f3 Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 23 Nov 2021 17:34:44 +0800 Subject: [PATCH 043/114] prepare test data and add test case --- core/pom.xml | 12 +++ core/src/main/job/mysql2tdengine.json | 24 +++-- .../datax/core/TestMysql2TDengine.java | 99 ++++++++++++++++++- 3 files changed, 128 insertions(+), 7 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 174a18d..3981cfc 100755 --- a/core/pom.xml +++ b/core/pom.xml @@ -97,6 +97,18 @@ groovy-all 2.1.9 + + mysql + mysql-connector-java + ${mysql.driver.version} + test + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + test + diff --git a/core/src/main/job/mysql2tdengine.json b/core/src/main/job/mysql2tdengine.json index 7978fbf..c936aa3 100644 --- a/core/src/main/job/mysql2tdengine.json +++ b/core/src/main/job/mysql2tdengine.json @@ -6,18 +6,18 @@ "name": "mysqlreader", "parameter": { "username": "root", - "password": "123456", + "password": "passw0rd", "column": [ "*" ], - "splitPk": "f1", + "splitPk": "station", "connection": [ { "table": [ "weather" ], "jdbcUrl": [ - "jdbc:mysql://192.168.56.105:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" ] } ] @@ -26,13 +26,25 @@ "writer": { "name": "tdenginewriter", "parameter": { - "host": "192.168.56.105", + "host": "127.0.0.1", "port": 6030, "dbname": "test", "user": "root", "password": "taosdata", - "table": "weather", - "batchSize": 1000 + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } } } } diff --git a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java index aeb0507..6edc185 100644 --- a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java +++ b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java @@ -1,10 +1,105 @@ package com.alibaba.datax.core; - +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import java.sql.*; +import java.util.*; +import java.util.Date; + +/** + * 测试从mysql到TD + */ public class TestMysql2TDengine { + @Test + public void genTestData() throws ClassNotFoundException, SQLException { + Class.forName("com.mysql.jdbc.Driver"); + + Connection conn = null; + Statement stmt = null; + ResultSet rs = null; + PreparedStatement pstmt = null; + + try { + conn = DriverManager.getConnection("jdbc:mysql://localhost/mysql?" + + "user=root&password=passw0rd"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists test"); + stmt.execute("use test"); + stmt.execute("drop table weather"); + stmt.execute("CREATE TABLE IF NOT EXISTS weather(station varchar(100), latitude DOUBLE, longtitude DOUBLE, `date` DATETIME, tmax INT, tmin INT)"); + pstmt = conn.prepareStatement("insert into weather(station, latitude, longtitude, `date`, tmax, tmin) values (?, ?, ?, ?, ?, ?)"); + genRandomData(pstmt); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException sqlEx) { + } // ignore + + rs = null; + } + + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException sqlEx) { + } // ignore + + stmt = null; + } + + if (pstmt != null) { + pstmt.close(); + } + } + + } + + private void genRandomData(PreparedStatement psmt) throws SQLException { + Random random = new Random(); + Calendar calendar = Calendar.getInstance(); + calendar.set(1990, 0, 1, 1, 0, 0); + List stations = Arrays.asList("STA", "STB", "STC"); + for (int i = 0; i < (10 * 100 * 24); i++) { + for (int j = 0; j < 3; j++) { + psmt.setString(1, stations.get(j)); + psmt.setDouble(2, random.nextDouble() * 1000); + psmt.setDouble(3, random.nextDouble() * 1000); + psmt.setTimestamp(4, new java.sql.Timestamp(calendar.getTime().getTime())); + psmt.setInt(5, random.nextInt(100)); + psmt.setInt(6, random.nextInt(100)); + psmt.addBatch(); + } + calendar.add(Calendar.MINUTE, 60); + if (i % 1000 == 0) { + psmt.executeBatch(); + } + } + psmt.executeBatch(); + } + + @Test + public void prepareTDengine() throws ClassNotFoundException, SQLException { + Class.forName("com.mysql.jdbc.Driver"); + + Connection conn = null; + Statement stmt = null; + + try { + conn = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists test"); + stmt.execute("drop stable if exists test.weather"); + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + @Test public void test() { System.out.println(System.getProperty("java.library.path")); @@ -17,4 +112,6 @@ public class TestMysql2TDengine { } } + + } \ No newline at end of file From 9b2d33b6e1761385d4bb8528409a64e7102bd9aa Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 23 Nov 2021 18:48:04 +0800 Subject: [PATCH 044/114] test case bug fix --- .../com/alibaba/datax/core/TestMysql2TDengine.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java index 6edc185..026b1db 100644 --- a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java +++ b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java @@ -1,12 +1,12 @@ package com.alibaba.datax.core; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.sql.*; -import java.util.*; -import java.util.Date; +import java.util.Arrays; +import java.util.Calendar; +import java.util.List; +import java.util.Random; /** * 测试从mysql到TD @@ -78,6 +78,7 @@ public class TestMysql2TDengine { psmt.executeBatch(); } } + // 有部分重复数据，不影响测试 psmt.executeBatch(); } @@ -91,7 +92,8 @@ public class TestMysql2TDengine { try { conn = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata"); stmt = conn.createStatement(); - stmt.execute("create database if not exists test"); + stmt.execute("drop database if exists test"); + stmt.execute("create database if not exists test keep 36500"); stmt.execute("drop stable if exists test.weather"); } finally { if (stmt != null) { From 5cc0fa3b1709441c42ac3e29bb09df39fe09b21b Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 23 Nov 2021 18:48:31 +0800 Subject: [PATCH 045/114] add mysql config demo to doc --- tdenginewriter/doc/tdenginewriter.md | 96 ++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 7 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 432b1fb..062fac2 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -181,8 +181,8 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea | fieldColumn | 字段列的列名和位置 | 否 | 无 | | | timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | -#### 3.3.3 自动建表规则 -##### 3.3.3.1 超级表创建规则 +#### 3.2.3 自动建表规则 +##### 3.2.3.1 超级表创建规则 如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： @@ -201,14 +201,14 @@ TAGS( ); ``` -##### 3.3.3.2 子表创建规则 +##### 3.2.3.2 子表创建规则 子表结果与超表相同，子表表名生成规则： 1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 2. 计算该字符串的 MD5 散列值 "md5_val"。 3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 -#### 3.3.4 用户提前建表 +#### 3.2.4 用户提前建表 如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： @@ -221,13 +221,13 @@ TAGS( ``` 那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 -#### 3.3.5 注意事项 +#### 3.2.5 注意事项 1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 3. 插件优先使用配置文件中指定的表结构。 -#### 3.3.6 类型转换 +#### 3.2.6 类型转换 | MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | | ---------------- | -------------- | ----------------- | @@ -238,6 +238,84 @@ TAGS( | boolean | Boolean | BOOL | | bytes | Bytes | BINARY | +### 3.3 从关系型数据库到TDengine +writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 + +#### 3.3.1 MySQL中表结构 +```sql +CREATE TABLE IF NOT EXISTS weather( + station varchar(100), + latitude DOUBLE, + longtitude DOUBLE, + `date` DATE, + TMAX int, + TMIN int +) +``` + +#### 3.3.2 配置文件示例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "passw0rd", + "column": [ + "*" + ], + "splitPk": "station", + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + ## 4 性能报告 @@ -313,4 +391,8 @@ TDengine要求每个表第一列是时间戳列，后边是普通字段，最后 ### 插件如何确定各列的数据类型？ -根据收到的第一批数据自动推断各列的类型。 \ No newline at end of file +根据收到的第一批数据自动推断各列的类型。 + +### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? + +因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file From 1bc24bfbd04229853e0de46a66bd4cfb7e96be14 Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 23 Nov 2021 18:55:29 +0800 Subject: [PATCH 046/114] update global readme add TDengine --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 37a2102..9d18a0a 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,8 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N | | FTP | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/ftpreader/doc/ftpreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/ftpwriter/doc/ftpwriter.md)| | | HDFS | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md)| | | Elasticsearch | | √ |[写](https://github.com/alibaba/DataX/blob/master/elasticsearchwriter/doc/elasticsearchwriter.md)| -| 时间序列数据库 | OpenTSDB | √ | |[读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)| +| 时间序列数据库 | TDengine | √ | √ |[读](https://github.com/taosdata/DataX/blob/master/tdenginereader/doc/tdenginereader.md) 、[写](https://github.com/taosdata/DataX/blob/master/tdenginewriter/doc/tdenginewriter.md)| +| | OpenTSDB | √ | |[读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)| | | TSDB | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md)| # 阿里云DataWorks数据集成 From f8aa7e90943f52b7a3c94d0ae0badb7f135105fe Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 23 Nov 2021 19:11:27 +0800 Subject: [PATCH 047/114] typo --- .../test/java/com/alibaba/datax/core/TestMysql2TDengine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java index 026b1db..c86f177 100644 --- a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java +++ b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.Random; /** - * 测试从mysql到TD + * 测试从mysql到TDengine */ public class TestMysql2TDengine { From a52b1909ca77e24490d7e4588b964bfbeee5cfce Mon Sep 17 00:00:00 2001 From: dingbo Date: Wed, 24 Nov 2021 07:58:25 +0800 Subject: [PATCH 048/114] no need to load driver class explicitly --- .../datax/core/TestMysql2TDengine.java | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java index c86f177..cab0629 100644 --- a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java +++ b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java @@ -14,12 +14,9 @@ import java.util.Random; public class TestMysql2TDengine { @Test - public void genTestData() throws ClassNotFoundException, SQLException { - Class.forName("com.mysql.jdbc.Driver"); - - Connection conn = null; + public void genTestData() throws SQLException { + Connection conn; Statement stmt = null; - ResultSet rs = null; PreparedStatement pstmt = null; try { @@ -33,15 +30,6 @@ public class TestMysql2TDengine { pstmt = conn.prepareStatement("insert into weather(station, latitude, longtitude, `date`, tmax, tmin) values (?, ?, ?, ?, ?, ?)"); genRandomData(pstmt); } finally { - if (rs != null) { - try { - rs.close(); - } catch (SQLException sqlEx) { - } // ignore - - rs = null; - } - if (stmt != null) { try { stmt.close(); @@ -53,6 +41,7 @@ public class TestMysql2TDengine { if (pstmt != null) { pstmt.close(); + pstmt = null; } } @@ -83,10 +72,8 @@ public class TestMysql2TDengine { } @Test - public void prepareTDengine() throws ClassNotFoundException, SQLException { - Class.forName("com.mysql.jdbc.Driver"); - - Connection conn = null; + public void prepareTDengine() throws SQLException { + Connection conn; Statement stmt = null; try { @@ -115,5 +102,4 @@ public class TestMysql2TDengine { } - } \ No newline at end of file From 64ba5e91a58b2525e072e43e07663f661e1dd92d Mon Sep 17 00:00:00 2001 From: dingbo Date: Wed, 24 Nov 2021 09:20:49 +0800 Subject: [PATCH 049/114] remove comment --- .../src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java index cab0629..4aaad64 100644 --- a/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java +++ b/core/src/test/java/com/alibaba/datax/core/TestMysql2TDengine.java @@ -67,7 +67,6 @@ public class TestMysql2TDengine { psmt.executeBatch(); } } - // 有部分重复数据，不影响测试 psmt.executeBatch(); } From c2027ed102085f6e789f77ef1342a7df1dc750f4 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 19 Nov 2021 16:38:55 +0800 Subject: [PATCH 050/114] migrate datax.py to python3, and compatible with python2 --- core/src/main/bin/datax.py | 114 +++++++++++++++++++++---------------- userGuid.md | 2 +- 2 files changed, 65 insertions(+), 51 deletions(-) diff --git a/core/src/main/bin/datax.py b/core/src/main/bin/datax.py index 1099ed3..4811ae8 100755 --- a/core/src/main/bin/datax.py +++ b/core/src/main/bin/datax.py @@ -1,23 +1,26 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- -import sys -import os -import signal -import subprocess -import time -import re -import socket -import json -from optparse import OptionParser -from optparse import OptionGroup -from string import Template import codecs +import json +import os import platform +import re +import signal +import socket +import subprocess +import sys +import time +from optparse import OptionGroup +from optparse import OptionParser +from string import Template + +ispy2 = sys.version_info.major == 2 def isWindows(): return platform.system() == 'Windows' + DATAX_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATAX_VERSION = 'DATAX-OPENSOURCE-3.0' @@ -52,13 +55,19 @@ def getLocalIp(): def suicide(signum, e): global child_process - print >> sys.stderr, "[Error] DataX receive unexpected signal %d, starts to suicide." % (signum) + if ispy2: + print >> sys.stderr, "[Error] DataX receive unexpected signal %d, starts to suicide." % (signum) + else: + print("[Error] DataX receive unexpected signal %d, starts to suicide." % (signum), sys.stderr) if child_process: child_process.send_signal(signal.SIGQUIT) time.sleep(1) child_process.kill() - print >> sys.stderr, "DataX Process was killed ! you did ?" + if ispy2: + print >> sys.stderr, "DataX Process was killed ! you did ?" + else: + print("DataX Process was killed ! you did ?", sys.stderr) sys.exit(RET_STATE["KILL"]) @@ -92,10 +101,10 @@ def getOptionParser(): 'if you have mutiple parameters: -p"-DtableName=your-table-name -DcolumnName=your-column-name".' 'Note: you should config in you job tableName with ${tableName}.') prodEnvOptionGroup.add_option("-r", "--reader", metavar="", - action="store", dest="reader",type="string", + action="store", dest="reader", type="string", help='View job config[reader] template, eg: mysqlreader,streamreader') prodEnvOptionGroup.add_option("-w", "--writer", metavar="", - action="store", dest="writer",type="string", + action="store", dest="writer", type="string", help='View job config[writer] template, eg: mysqlwriter,streamwriter') parser.add_option_group(prodEnvOptionGroup) @@ -108,45 +117,50 @@ def getOptionParser(): parser.add_option_group(devEnvOptionGroup) return parser + def generateJobConfigTemplate(reader, writer): - readerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n" % (reader,reader,reader) - writerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n " % (writer,writer,writer) - print readerRef - print writerRef + readerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n" % ( + reader, reader, reader) + writerRef = "Please refer to the %s document:\n https://github.com/alibaba/DataX/blob/master/%s/doc/%s.md \n " % ( + writer, writer, writer) + print(readerRef) + print(writerRef) jobGuid = 'Please save the following configuration as a json file and use\n python {DATAX_HOME}/bin/datax.py {JSON_FILE_NAME}.json \nto run the job.\n' - print jobGuid - jobTemplate={ - "job": { - "setting": { - "speed": { - "channel": "" - } - }, - "content": [ - { - "reader": {}, - "writer": {} - } - ] - } + print(jobGuid) + jobTemplate = { + "job": { + "setting": { + "speed": { + "channel": "" + } + }, + "content": [ + { + "reader": {}, + "writer": {} + } + ] + } } - readerTemplatePath = "%s/plugin/reader/%s/plugin_job_template.json" % (DATAX_HOME,reader) - writerTemplatePath = "%s/plugin/writer/%s/plugin_job_template.json" % (DATAX_HOME,writer) + readerTemplatePath = "%s/plugin/reader/%s/plugin_job_template.json" % (DATAX_HOME, reader) + writerTemplatePath = "%s/plugin/writer/%s/plugin_job_template.json" % (DATAX_HOME, writer) try: - readerPar = readPluginTemplate(readerTemplatePath); - except Exception, e: - print "Read reader[%s] template error: can\'t find file %s" % (reader,readerTemplatePath) + readerPar = readPluginTemplate(readerTemplatePath) + except: + print("Read reader[%s] template error: can\'t find file %s" % (reader, readerTemplatePath)) try: - writerPar = readPluginTemplate(writerTemplatePath); - except Exception, e: - print "Read writer[%s] template error: : can\'t find file %s" % (writer,writerTemplatePath) - jobTemplate['job']['content'][0]['reader'] = readerPar; - jobTemplate['job']['content'][0]['writer'] = writerPar; - print json.dumps(jobTemplate, indent=4, sort_keys=True) + writerPar = readPluginTemplate(writerTemplatePath) + except: + print("Read writer[%s] template error: : can\'t find file %s" % (writer, writerTemplatePath)) + jobTemplate['job']['content'][0]['reader'] = readerPar + jobTemplate['job']['content'][0]['writer'] = writerPar + print(json.dumps(jobTemplate, indent=4, sort_keys=True)) + def readPluginTemplate(plugin): with open(plugin, 'r') as f: - return json.load(f) + return json.load(f) + def isUrl(path): if not path: @@ -168,7 +182,7 @@ def buildStartCommand(options, args): if options.remoteDebug: tempJVMCommand = tempJVMCommand + " " + REMOTE_DEBUG_CONFIG - print 'local ip: ', getLocalIp() + print('local ip: ', getLocalIp()) if options.loglevel: tempJVMCommand = tempJVMCommand + " " + ("-Dloglevel=%s" % (options.loglevel)) @@ -198,11 +212,11 @@ def buildStartCommand(options, args): def printCopyright(): - print ''' + print(''' DataX (%s), From Alibaba ! Copyright (C) 2010-2017, Alibaba Group. All Rights Reserved. -''' % DATAX_VERSION +''' % DATAX_VERSION) sys.stdout.flush() @@ -211,7 +225,7 @@ if __name__ == "__main__": parser = getOptionParser() options, args = parser.parse_args(sys.argv[1:]) if options.reader is not None and options.writer is not None: - generateJobConfigTemplate(options.reader,options.writer) + generateJobConfigTemplate(options.reader, options.writer) sys.exit(RET_STATE['OK']) if len(args) != 1: parser.print_help() diff --git a/userGuid.md b/userGuid.md index 153c811..16771a5 100644 --- a/userGuid.md +++ b/userGuid.md @@ -10,7 +10,7 @@ DataX本身作为数据同步框架，将不同数据源的同步抽象为从源 - Linux - [JDK(1.8以上，推荐1.8) ](http://www.oracle.com/technetwork/cn/java/javase/downloads/index.html) -- [Python(推荐Python2.6.X) ](https://www.python.org/downloads/) +- [Python(2或3都可以) ](https://www.python.org/downloads/) - [Apache Maven 3.x](https://maven.apache.org/download.cgi) (Compile DataX) # Quick Start From 9587f9b93be0833b35d512649fced60c9593c4a2 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 11 Nov 2021 15:24:09 +0800 Subject: [PATCH 051/114] add option authDB to mongodb reader doc --- mongodbreader/doc/mongodbreader.md | 1 + 1 file changed, 1 insertion(+) diff --git a/mongodbreader/doc/mongodbreader.md b/mongodbreader/doc/mongodbreader.md index b61493e..99d2573 100644 --- a/mongodbreader/doc/mongodbreader.md +++ b/mongodbreader/doc/mongodbreader.md @@ -127,6 +127,7 @@ MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的J * address： MongoDB的数据地址信息，因为MonogDB可能是个集群，则ip端口信息需要以Json数组的形式给出。【必填】 * userName：MongoDB的用户名。【选填】 * userPassword： MongoDB的密码。【选填】 +* authDb: MongoDB认证数据库【选填】 * collectionName： MonogoDB的集合名。【必填】 * column：MongoDB的文档列名。【必填】 * name：Column的名字。【必填】 From ef6c53e7e22c74efc2b03dd1babad21bef501e18 Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 29 Nov 2021 13:15:49 +0800 Subject: [PATCH 052/114] add tdengine weriter --- tdenginewriter/doc/tdenginewriter.md | 398 ++++++++++++++++++ tdenginewriter/pom.xml | 107 +++++ tdenginewriter/src/main/assembly/package.xml | 34 ++ .../writer/tdenginewriter/DataHandler.java | 12 + .../tdenginewriter/DataHandlerFactory.java | 10 + .../tdenginewriter/DefaultDataHandler.java | 105 +++++ .../tdenginewriter/JDBCBatchWriter.java | 244 +++++++++++ .../writer/tdenginewriter/JniConnection.java | 89 ++++ .../plugin/writer/tdenginewriter/Key.java | 14 + .../plugin/writer/tdenginewriter/Msg.java | 20 + .../tdenginewriter/OpentsdbDataHandler.java | 99 +++++ .../writer/tdenginewriter/SchemaManager.java | 271 ++++++++++++ .../writer/tdenginewriter/TDengineWriter.java | 91 ++++ .../TDengineWriterErrorCode.java | 32 ++ ...ugin_writer_tdenginewriter_JniConnection.h | 105 +++++ tdenginewriter/src/main/resources/plugin.json | 9 + .../main/resources/plugin_job_template.json | 10 + .../resources/tdenginewritermsg.properties | 6 + .../tdenginewritermsg_en_US.properties | 6 + .../tdenginewritermsg_zh_CN.properties | 6 + .../tdenginewriter/JniConnectionTest.java | 21 + .../writer/tdenginewriter/MessageTest.java | 25 ++ .../tdenginewriter/TDengineWriterTest.java | 31 ++ 23 files changed, 1745 insertions(+) create mode 100644 tdenginewriter/doc/tdenginewriter.md create mode 100644 tdenginewriter/pom.xml create mode 100644 tdenginewriter/src/main/assembly/package.xml create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h create mode 100644 tdenginewriter/src/main/resources/plugin.json create mode 100644 tdenginewriter/src/main/resources/plugin_job_template.json create mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg.properties create mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties create mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md new file mode 100644 index 0000000..062fac2 --- /dev/null +++ b/tdenginewriter/doc/tdenginewriter.md @@ -0,0 +1,398 @@ +# DataX TDengineWriter + +## 1 快速介绍 + +TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 + +## 2 实现原理 + +TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： + +1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 + +2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 + +这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 + +## 3 功能说明 +### 3.1 从OpenTSDB到TDengine +#### 3.1.1 配置样例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather_temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.1.180", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +#### 3.1.2 参数说明 + +| 参数 | 描述 | 是否必选 | 默认值 | +| --------- | -------------------- | -------- | -------- | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbname | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1 | + + +#### 3.1.3 类型转换 + +目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： + +| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | + +### 3.2 从MongoDB到TDengine + +#### 3.2.1 配置样例 +```json +{ + "job": { + "setting": { + "speed": { + "channel": 2 + } + }, + "content": [ + { + "reader": { + "name": "mongodbreader", + "parameter": { + "address": [ + "127.0.0.1:27017" + ], + "userName": "user", + "mechanism": "SCRAM-SHA-1", + "userPassword": "password", + "authDb": "admin", + "dbName": "test", + "collectionName": "stock", + "column": [ + { + "name": "stockID", + "type": "string" + }, + { + "name": "tradeTime", + "type": "date" + }, + { + "name": "lastPrice", + "type": "double" + }, + { + "name": "askPrice1", + "type": "double" + }, + { + "name": "bidPrice1", + "type": "double" + }, + { + "name": "volume", + "type": "int" + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "localhost", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "stable": "stock", + "tagColumn": { + "industry": "energy", + "stockID": 0 + }, + "fieldColumn": { + "lastPrice": 2, + "askPrice1": 3, + "bidPrice1": 4, + "volume": 5 + }, + "timestampColumn": { + "tradeTime": 1 + } + } + } + } + ] + } +} +``` + +**注：本配置的writer部分同样适用于关系型数据库** + + +#### 3.2.2 参数说明 +| 参数 | 描述 | 是否必选 | 默认值 | 备注 | +| --------------- | -------------------- | ---------------- | -------- | ------------------ | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbname | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1000 | +| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | +| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | +| fieldColumn | 字段列的列名和位置 | 否 | 无 | | +| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | + +#### 3.2.3 自动建表规则 +##### 3.2.3.1 超级表创建规则 + +如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
+数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： + +```sql +CREATE STABLE IF NOT EXISTS market_snapshot ( + tadetime TIMESTAMP, + lastprice DOUBLE, + askprice1 DOUBLE, + bidprice1 DOUBLE, + volume INT +) +TAGS( + industry NCHAR(64), + stockID NCHAR(64) +); +``` + +##### 3.2.3.2 子表创建规则 + +子表结果与超表相同，子表表名生成规则： +1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 +2. 计算该字符串的 MD5 散列值 "md5_val"。 +3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 + +#### 3.2.4 用户提前建表 + +如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 +此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： +``` + Field | Type | Length | Note | +================================================================================= + ts | TIMESTAMP | 8 | | + current | DOUBLE | 8 | | + location | BINARY | 10 | TAG | +``` +那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 + +#### 3.2.5 注意事项 + +1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 +2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 +3. 插件优先使用配置文件中指定的表结构。 + +#### 3.2.6 类型转换 + +| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| int, Long | Long | BIGINT | +| double | Double | DOUBLE | +| string, array | String | NCHAR(64) | +| date | Date | TIMESTAMP | +| boolean | Boolean | BOOL | +| bytes | Bytes | BINARY | + +### 3.3 从关系型数据库到TDengine +writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 + +#### 3.3.1 MySQL中表结构 +```sql +CREATE TABLE IF NOT EXISTS weather( + station varchar(100), + latitude DOUBLE, + longtitude DOUBLE, + `date` DATE, + TMAX int, + TMIN int +) +``` + +#### 3.3.2 配置文件示例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "passw0rd", + "column": [ + "*" + ], + "splitPk": "station", + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数 | DataX速度(Rec/s) | DataX流量(MB/s) | DataX机器网卡流出流量(MB/s) | DataX机器运行负载 | DB网卡进入流量(MB/s) | DB运行负载 | DB TPS | +| ------ | ---------------- | --------------- | --------------------------- | ----------------- | -------------------- | ---------- | ------ | +| 1 | | | | | | | | +| 4 | | | | | | | | +| 8 | | | | | | | | +| 16 | | | | | | | | +| 32 | | | | | | | | + +说明： + +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 +3. 16通道，4096批量提交时，出现 full gc 2次。 + +#### 4.2.4 性能测试小结 + + +## 5 约束限制 + +1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 +2. 标签列不能包含null值，如果包含会被过滤掉。 + +## FAQ + +### 如何选取要同步的数据的范围？ + +数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 + +### 如何一次导入多张源表？ + +如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 + +### 一张源表导入之后对应TDengine中多少张表？ + +这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超表就有多少子表。 + +### 源表和目标表的字段顺序一致吗？ + +TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表是自动调整。 + +### 插件如何确定各列的数据类型？ + +根据收到的第一批数据自动推断各列的类型。 + +### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? + +因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml new file mode 100644 index 0000000..054f2ef --- /dev/null +++ b/tdenginewriter/pom.xml @@ -0,0 +1,107 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + com.alibaba.datax.tdenginewriter + tdenginewriter + 0.0.1-SNAPSHOT + + + 8 + 8 + + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + + + + junit + junit + ${junit-version} + test + + + org.apache.commons + commons-lang3 + ${commons-lang3-version} + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + + + **/*Test.java + + + + + true + + + + + + + \ No newline at end of file diff --git a/tdenginewriter/src/main/assembly/package.xml b/tdenginewriter/src/main/assembly/package.xml new file mode 100644 index 0000000..d3b75ea --- /dev/null +++ b/tdenginewriter/src/main/assembly/package.xml @@ -0,0 +1,34 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/writer/tdenginewriter + + + target/ + + tdenginewriter-0.0.1-SNAPSHOT.jar + + plugin/writer/tdenginewriter + + + + + + false + plugin/writer/tdenginewriter/libs + runtime + + + diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java new file mode 100644 index 0000000..dcc3ca8 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java @@ -0,0 +1,12 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.plugin.RecordReceiver; + +import com.alibaba.datax.common.plugin.TaskPluginCollector; + +import java.util.Properties; + +public interface DataHandler { + + long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector); +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java new file mode 100644 index 0000000..1f740d7 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java @@ -0,0 +1,10 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public class DataHandlerFactory { + + public static DataHandler build(String peerPluginName) { + if (peerPluginName.equals("opentsdbreader")) + return new OpentsdbDataHandler(); + return new DefaultDataHandler(); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java new file mode 100644 index 0000000..91c2b7e --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -0,0 +1,105 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.taosdata.jdbc.TSDBPreparedStatement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +/** + * 默认DataHandler + */ +public class DefaultDataHandler implements DataHandler { + private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); + + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + @Override + public long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector) { + SchemaManager schemaManager = new SchemaManager(properties); + if (!schemaManager.configValid()) { + return 0; + } + + try { + Connection conn = getTaosConnection(properties); + if (conn == null) { + return 0; + } + if (schemaManager.shouldGuessSchema()) { + // 无法从配置文件获取表结构信息，尝试从数据库获取 + LOG.info(Msg.get("try_get_schema_from_db")); + boolean success = schemaManager.getFromDB(conn); + if (!success) { + return 0; + } + } else { + + } + int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, "1000")); + if (batchSize < 5) { + // batchSize太小，会增加自动类型推断错误的概率，建议改大后重试 + LOG.error(Msg.get("batch_size_too_small")); + return 0; + } + return write(lineReceiver, conn, batchSize, schemaManager, collector); + } catch (Exception e) { + LOG.error("write failed " + e.getMessage()); + e.printStackTrace(); + } + return 0; + } + + + private Connection getTaosConnection(Properties properties) throws SQLException { + // 检查必要参数 + String host = properties.getProperty(Key.HOST); + String port = properties.getProperty(Key.PORT); + String dbname = properties.getProperty(Key.DBNAME); + String user = properties.getProperty(Key.USER); + String password = properties.getProperty(Key.PASSWORD); + if (host == null || port == null || dbname == null || user == null || password == null) { + String keys = String.join(" ", Key.HOST, Key.PORT, Key.DBNAME, Key.USER, Key.PASSWORD); + LOG.error("Required options missing, please check: " + keys); + return null; + } + String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", host, port, dbname, user, password); + LOG.info("TDengine connection established, host:{} port:{} dbname:{} user:{}", host, port, dbname, user); + return DriverManager.getConnection(jdbcUrl); + } + + /** + * 使用SQL批量写入
+ * + * @return 成功写入记录数 + * @throws SQLException + */ + private long write(RecordReceiver lineReceiver, Connection conn, int batchSize, SchemaManager scm, TaskPluginCollector collector) throws SQLException { + Record record = lineReceiver.getFromReader(); + if (record == null) { + return 0; + } + String pq = String.format("INSERT INTO ? USING %s TAGS(%s) (%s) values (%s)", scm.getStable(), scm.getTagValuesPlaceHolder(), scm.getJoinedFieldNames(), scm.getFieldValuesPlaceHolder()); + LOG.info("Prepared SQL: {}", pq); + try (TSDBPreparedStatement stmt = (TSDBPreparedStatement) conn.prepareStatement(pq)) { + JDBCBatchWriter batchWriter = new JDBCBatchWriter(conn, stmt, scm, batchSize, collector); + do { + batchWriter.append(record); + } while ((record = lineReceiver.getFromReader()) != null); + batchWriter.flush(); + return batchWriter.getCount(); + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java new file mode 100644 index 0000000..53ab9bb --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java @@ -0,0 +1,244 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.taosdata.jdbc.TSDBPreparedStatement; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * 使用JDBC原生写入接口批量写入。
+ * 有两个限制条件导致批量写入的代码逻辑过于复杂，以至于需要开发新的类来封装。
+ * 1. 用户必须提前把需要批量写入的数据搜集到ArrayList中 + * 2. 每批写入的表名必须相同。 + * 这个类的实现逻辑是： + * 1. 先把属于同一子表的Record缓存起来 + * 2. 缓存的数量达到batchSize阈值，自动执行一次批量写入 + * 3. 最后一批数据需要用户手动flush才能写入 + */ +public class JDBCBatchWriter { + public static final Logger LOG = LoggerFactory.getLogger(JDBCBatchWriter.class); + private TSDBPreparedStatement stmt; + private SchemaManager scm; + private Connection conn; + private int batchSize; + private TaskPluginCollector collector; + + // 缓存Record， key为tableName + Map> buf = new HashMap<>(); + // 缓存表的标签值， key为tableName + Map tableTagValues = new HashMap<>(); + private long sucCount = 0; + private final int tsColIndex; + private List fieldList; + // 每个record至少应该包含的列数，用于校验数据 + private int minColNum = 0; + private Map fieldIndexMap; + private List fieldTypes = null; + + public JDBCBatchWriter(Connection conn, TSDBPreparedStatement stmt, SchemaManager scm, int batchSize, TaskPluginCollector collector) { + this.conn = conn; + this.stmt = stmt; + this.scm = scm; + this.batchSize = batchSize; + this.collector = collector; + this.tsColIndex = scm.getTsColIndex(); + this.fieldList = scm.getFieldList(); + this.fieldIndexMap = scm.getFieldIndexMap(); + this.minColNum = 1 + fieldList.size() + scm.getDynamicTagCount(); + + } + + public void initFiledTypesAndTargetTable(List records) throws SQLException { + if (fieldTypes != null) { + return; + } + guessFieldTypes(records); + if (scm.shouldCreateTable()) { + scm.createSTable(conn, fieldTypes); + } + } + + public void append(Record record) throws SQLException { + int columnNum = record.getColumnNumber(); + if (columnNum < minColNum) { + // 实际列数小于期望列数 + collector.collectDirtyRecord(record, Msg.get("column_number_error")); + return; + } + String[] tagValues = scm.getTagValuesFromRecord(record); + if (tagValues == null) { + // 标签列包含null + collector.collectDirtyRecord(record, Msg.get("tag_value_error")); + return; + } + if (!scm.hasTimestamp(record)) { + // 时间戳列为null或类型错误 + collector.collectDirtyRecord(record, Msg.get("ts_value_error")); + return; + } + String tableName = scm.computeTableName(tagValues); + if (buf.containsKey(tableName)) { + List lis = buf.get(tableName); + lis.add(record); + if (lis.size() == batchSize) { + if (fieldTypes == null) { + initFiledTypesAndTargetTable(lis); + } + executeBatch(tableName); + lis.clear(); + } + } else { + List lis = new ArrayList<>(batchSize); + lis.add(record); + buf.put(tableName, lis); + tableTagValues.put(tableName, tagValues); + } + } + + /** + * 只有String类型比较特别，测试发现值为null的列会转成String类型。所以Column的类型为String并不代表这一列的类型真的是String。 + * + * @param records + */ + private void guessFieldTypes(List records) { + fieldTypes = new ArrayList<>(fieldList.size()); + for (int i = 0; i < fieldList.size(); ++i) { + int colIndex = fieldIndexMap.get(fieldList.get(i)); + boolean ok = false; + for (int j = 0; j < records.size() && !ok; ++j) { + Column column = records.get(j).getColumn(colIndex); + Column.Type type = column.getType(); + switch (type) { + case LONG: + case DOUBLE: + case DATE: + case BOOL: + case BYTES: + if (column.getRawData() != null) { + fieldTypes.add(type); + ok = true; + } + break; + case STRING: + // 只有非null且非空的String列，才会被真的当作String类型。 + String value = column.asString(); + if (value != null && !"".equals(value)) { + fieldTypes.add(type); + ok = true; + } + break; + default: + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, fieldTypes.get(i).toString()); + } + } + if (!ok) { + // 根据采样的%d条数据，无法推断第%d列的数据类型 + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, String.format(Msg.get("infer_column_type_error"), records.size(), i + 1)); + } + } + LOG.info("Field Types: {}", fieldTypes); + } + + /** + * 执行单表批量写入 + * + * @param tableName + * @throws SQLException + */ + private void executeBatch(String tableName) throws SQLException { + // 表名 + stmt.setTableName(tableName); + List records = buf.get(tableName); + // 标签 + String[] tagValues = tableTagValues.get(tableName); + LOG.debug("executeBatch {}", String.join(",", tagValues)); + for (int i = 0; i < tagValues.length; ++i) { + stmt.setTagNString(i, tagValues[i]); + } + // 时间戳 + ArrayList tsList = records.stream().map(r -> r.getColumn(tsColIndex).asDate().getTime()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setTimestamp(0, tsList); + // 字段 + for (int i = 0; i < fieldList.size(); ) { + String fieldName = fieldList.get(i); + int index = fieldIndexMap.get(fieldName); + switch (fieldTypes.get(i)) { + case LONG: + ArrayList lisLong = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setLong(++i, lisLong); + break; + case DOUBLE: + ArrayList lisDouble = records.stream().map(r -> r.getColumn(index).asDouble()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setDouble(++i, lisDouble); + break; + case STRING: + ArrayList lisString = records.stream().map(r -> r.getColumn(index).asString()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setNString(++i, lisString, 64); + break; + case DATE: + ArrayList lisTs = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setTimestamp(++i, lisTs); + break; + case BOOL: + ArrayList lisBool = records.stream().map(r -> r.getColumn(index).asBoolean()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setBoolean(++i, lisBool); + break; + case BYTES: + ArrayList lisBytes = records.stream().map(r -> r.getColumn(index).asString()).collect(Collectors.toCollection(ArrayList::new)); + stmt.setString(++i, lisBytes, 64); + break; + default: + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, fieldTypes.get(i).toString()); + } + } + // 执行 + stmt.columnDataAddBatch(); + stmt.columnDataExecuteBatch(); + // 更新计数器 + sucCount += records.size(); + } + + /** + * 把缓存的Record全部写入 + */ + public void flush() throws SQLException { + if (fieldTypes == null) { + List records = new ArrayList<>(); + for (List lis : buf.values()) { + records.addAll(lis); + if (records.size() > 100) { + break; + } + } + if (records.size() > 0) { + initFiledTypesAndTargetTable(records); + } else { + return; + } + } + for (String tabName : buf.keySet()) { + if (buf.get(tabName).size() > 0) { + executeBatch(tabName); + } + } + stmt.columnDataCloseBatch(); + } + + /** + * @return 成功写入的数据量 + */ + public long getCount() { + return sucCount; + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java new file mode 100644 index 0000000..0aabe32 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java @@ -0,0 +1,89 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import java.util.Properties; + +public class JniConnection { + + private static final long JNI_NULL_POINTER = 0L; + private static final int JNI_SUCCESSFUL = 0; + public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; + public static final String PROPERTY_KEY_LOCALE = "locale"; + public static final String PROPERTY_KEY_CHARSET = "charset"; + public static final String PROPERTY_KEY_TIME_ZONE = "timezone"; + + private long conn; + + static { + System.loadLibrary("taos"); + } + + public JniConnection(Properties props) throws Exception { + initImp(props.getProperty(PROPERTY_KEY_CONFIG_DIR, null)); + + String locale = props.getProperty(PROPERTY_KEY_LOCALE); + if (setOptions(0, locale) < 0) { + throw new Exception("Failed to set locale: " + locale + ". System default will be used."); + } + String charset = props.getProperty(PROPERTY_KEY_CHARSET); + if (setOptions(1, charset) < 0) { + throw new Exception("Failed to set charset: " + charset + ". System default will be used."); + } + String timezone = props.getProperty(PROPERTY_KEY_TIME_ZONE); + if (setOptions(2, timezone) < 0) { + throw new Exception("Failed to set timezone: " + timezone + ". System default will be used."); + } + } + + public void open(String host, int port, String dbname, String user, String password) throws Exception { + if (this.conn != JNI_NULL_POINTER) { + close(); + this.conn = JNI_NULL_POINTER; + } + + this.conn = connectImp(host, port, dbname, user, password); + if (this.conn == JNI_NULL_POINTER) { + String errMsg = getErrMsgImp(0); + throw new Exception(errMsg); + } + } + + public void insertOpentsdbJson(String json) throws Exception { + if (this.conn == JNI_NULL_POINTER) { + throw new Exception("JNI connection is NULL"); + } + + long result = insertOpentsdbJson(json, this.conn); + int errCode = getErrCodeImp(this.conn, result); + if (errCode != JNI_SUCCESSFUL) { + String errMsg = getErrMsgImp(result); + freeResultSetImp(this.conn, result); + throw new Exception(errMsg); + } + freeResultSetImp(this.conn, result); + } + + public void close() throws Exception { + int code = this.closeConnectionImp(this.conn); + if (code != 0) { + throw new Exception("JNI closeConnection failed"); + } + this.conn = JNI_NULL_POINTER; + } + + private static native void initImp(String configDir); + + private static native int setOptions(int optionIndex, String optionValue); + + private native long connectImp(String host, int port, String dbName, String user, String password); + + private native int getErrCodeImp(long connection, long pSql); + + private native String getErrMsgImp(long pSql); + + private native void freeResultSetImp(long connection, long pSql); + + private native int closeConnectionImp(long connection); + + private native long insertOpentsdbJson(String json, long pSql); + +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java new file mode 100644 index 0000000..090a799 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -0,0 +1,14 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public class Key { + public static final String HOST = "host"; + public static final String PORT = "port"; + public static final String DBNAME = "dbname"; + public static final String USER = "user"; + public static final String PASSWORD = "password"; + public static final String BATCH_SIZE = "batchSize"; + public static final String STABLE = "stable"; + public static final String TAG_COLUMN = "tagColumn"; + public static final String FIELD_COLUMN = "fieldColumn"; + public static final String TIMESTAMP_COLUMN = "timestampColumn"; +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java new file mode 100644 index 0000000..89730d3 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java @@ -0,0 +1,20 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import java.util.Locale; +import java.util.ResourceBundle; + +/** + * i18n message util + */ +public class Msg { + private static ResourceBundle bundle; + + static { + bundle = ResourceBundle.getBundle("tdenginewritermsg", Locale.getDefault()); + } + + public static String get(String key) { + return bundle.getString(key); + } + +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java new file mode 100644 index 0000000..e1b8f5d --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java @@ -0,0 +1,99 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; + +public class OpentsdbDataHandler implements DataHandler { + private static final Logger LOG = LoggerFactory.getLogger(OpentsdbDataHandler.class); + private static final String DEFAULT_BATCH_SIZE = "1"; + + @Override + public long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector) { + // opentsdb json protocol use JNI and schemaless API to write + String host = properties.getProperty(Key.HOST); + int port = Integer.parseInt(properties.getProperty(Key.PORT)); + String dbname = properties.getProperty(Key.DBNAME); + String user = properties.getProperty(Key.USER); + String password = properties.getProperty(Key.PASSWORD); + + JniConnection conn = null; + long count = 0; + try { + conn = new JniConnection(properties); + conn.open(host, port, dbname, user, password); + LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); + int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, DEFAULT_BATCH_SIZE)); + count = writeOpentsdb(lineReceiver, conn, batchSize); + } catch (Exception e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + } finally { + try { + if (conn != null) + conn.close(); + } catch (Exception e) { + e.printStackTrace(); + } + LOG.info("TDengine connection closed"); + } + + return count; + } + + private long writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn, int batchSize) { + long recordIndex = 1; + try { + Record record; + StringBuilder sb = new StringBuilder(); + while ((record = lineReceiver.getFromReader()) != null) { + if (batchSize == 1) { + String jsonData = recordToString(record); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + } else if (recordIndex % batchSize == 1) { + sb.append("[").append(recordToString(record)).append(","); + } else if (recordIndex % batchSize == 0) { + sb.append(recordToString(record)).append("]"); + String jsonData = sb.toString(); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + sb.delete(0, sb.length()); + } else { + sb.append(recordToString(record)).append(","); + } + recordIndex++; + } + if (sb.length() != 0 && sb.charAt(0) == '[') { + String jsonData = sb.deleteCharAt(sb.length() - 1).append("]").toString(); + LOG.debug(">>> " + jsonData); + conn.insertOpentsdbJson(jsonData); + } + } catch (Exception e) { + LOG.error("TDengineWriter ERROR: " + e.getMessage()); + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); + } + return recordIndex - 1; + } + + private String recordToString(Record record) { + int recordLength = record.getColumnNumber(); + if (0 == recordLength) { + return ""; + } + Column column; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < recordLength; i++) { + column = record.getColumn(i); + sb.append(column.asString()).append("\t"); + } + sb.setLength(sb.length() - 1); + return sb.toString(); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java new file mode 100644 index 0000000..d67a658 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -0,0 +1,271 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.*; +import java.util.stream.Collectors; + +public class SchemaManager { + private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class); + + private String stable; // 目标超表名 + private Map fixedTagValue = new HashMap<>(); // 固定标签值 标签名 -> 标签值 + private Map tagIndexMap = new HashMap<>(); // 动态标签值 标签名 -> 列索引 + private Map fieldIndexMap = new HashMap<>(); // 字段名 -> 字段索引 + private String tsColName; // 时间戳列名 + private int tsColIndex = -1; // 时间戳列索引 + private List fieldList = new ArrayList<>(); + private List tagList = new ArrayList<>(); + private boolean canInferSchemaFromConfig = false; + + + public SchemaManager() { + } + + public SchemaManager(Properties properties) { + getFromConfig(properties); + } + + private String mapDataxType(Column.Type type) { + switch (type) { + case LONG: + return "BIGINT"; + case DOUBLE: + return "DOUBLE"; + case STRING: + return "NCHAR(64)"; + case DATE: + return "TIMESTAMP"; + case BOOL: + return "BOOL"; + case BYTES: + return "BINARY(64)"; + default: + throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, type.toString()); + } + } + + public void setStable(String stable) { + stable = stable; + } + + public String getStable() { + return stable; + } + + private void getFromConfig(Properties properties) { + stable = properties.getProperty(Key.STABLE); + if (stable == null) { + LOG.error("Config error: no stable"); + return; + } + for (Object key : properties.keySet()) { + String k = (String) key; + String v = properties.getProperty(k); + + String[] ps = k.split("\\."); + if (ps.length == 1) { + continue; + } + if (k.startsWith(Key.TAG_COLUMN)) { + String tagName = ps[1]; + try { + Integer tagIndex = Integer.parseInt(v); + this.tagIndexMap.put(tagName, tagIndex); + tagList.add(tagName); + } catch (NumberFormatException e) { + fixedTagValue.put(tagName, v); + tagList.add(tagName); + } + } else if (k.startsWith(Key.FIELD_COLUMN)) { + String fieldName = ps[1]; + Integer fileIndex = Integer.parseInt(v); + fieldIndexMap.put(fieldName, fileIndex); + } else if (k.startsWith(Key.TIMESTAMP_COLUMN)) { + tsColName = ps[1]; + tsColIndex = Integer.parseInt(v); + } + } + List sortedFieldName = fieldIndexMap.entrySet().stream().sorted((x, y) -> x.getValue().compareTo(y.getValue())).map(e -> e.getKey()).collect(Collectors.toList()); + fieldList.addAll(sortedFieldName); // 排序的目的是保证自动建表时列的顺序和输入数据的列的顺序保持一致 + canInferSchemaFromConfig = tsColIndex > -1 && !(fixedTagValue.isEmpty() && tagIndexMap.isEmpty()) && !fieldIndexMap.isEmpty(); + LOG.info("Config file parsed result：fixedTags=[{}] ,tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", fixedTagValue.keySet()), String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); + } + + public boolean shouldGuessSchema() { + return !canInferSchemaFromConfig; + } + + public boolean shouldCreateTable() { + return canInferSchemaFromConfig; + } + + public boolean configValid() { + boolean valid = (tagList.size() > 0 && fieldList.size() > 0 && tsColIndex > -1) || (tagList.size() == 0 && fieldList.size() == 0 && tsColIndex == -1); + if (!valid) { + LOG.error("Config error: tagColumn, fieldColumn and timestampColumn must be present together or absent together."); + } + return valid; + } + + /** + * 通过执行`describe dbname.stable`命令，获取表的schema.
+ * describe命名返回有4列内容，分布是：Field,Type,Length,Note
+ * + * @return 成功返回true，如果超表不存在或其他错误则返回false + */ + public boolean getFromDB(Connection conn) { + try { + List stables = getSTables(conn); + if (!stables.contains(stable)) { + LOG.error("super table {} not exist， fail to get schema from database.", stable); + return false; + } + } catch (SQLException e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + return false; + } + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("describe " + stable); + int colIndex = 0; + while (rs.next()) { + String name = rs.getString(1); + String type = rs.getString(2); + String note = rs.getString(4); + if ("TIMESTAMP".equals(type)) { + tsColName = name; + tsColIndex = colIndex; + } else if ("TAG".equals(note)) { + tagIndexMap.put(name, colIndex); + tagList.add(name); + } else { + fieldIndexMap.put(name, colIndex); + fieldList.add(name); + } + colIndex++; + } + LOG.info("table info：tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); + return true; + } catch (SQLException e) { + LOG.error(e.getMessage()); + e.printStackTrace(); + return false; + } + } + + public static List getSTables(Connection conn) throws SQLException { + List stables = new ArrayList<>(); + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("show stables"); + while (rs.next()) { + String name = rs.getString(1); + stables.add(name); + } + } + return stables; + } + + public void createSTable(Connection conn, List fieldTypes) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE STABLE IF NOT EXISTS ").append(stable).append("("); + sb.append(tsColName).append(" ").append("TIMESTAMP,"); + for (int i = 0; i < fieldList.size(); ++i) { + String fieldName = fieldList.get(i); + Column.Type dxType = fieldTypes.get(i); + sb.append(fieldName).append(' '); + String tdType = mapDataxType(dxType); + sb.append(tdType).append(','); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(") TAGS("); + for (String tagName : tagList) { + sb.append(tagName).append(" NCHAR(64),"); + } + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + String q = sb.toString(); + LOG.info("run sql：" + q); + try (Statement stmt = conn.createStatement()) { + stmt.execute(q); + } + } + + public String[] getTagValuesFromRecord(Record record) { + String[] tagValues = new String[tagList.size()]; + for (int i = 0; i < tagList.size(); ++i) { + if (fixedTagValue.containsKey(tagList.get(i))) { + tagValues[i] = fixedTagValue.get(tagList.get(i)); + } else { + int tagIndex = tagIndexMap.get(tagList.get(i)); + tagValues[i] = record.getColumn(tagIndex).asString(); + } + if (tagValues[i] == null) { + return null; + } + } + return tagValues; + } + + public boolean hasTimestamp(Record record) { + Column column = record.getColumn(tsColIndex); + if (column.getType() == Column.Type.DATE && column.asDate() != null) { + return true; + } else { + return false; + } + } + + public Map getFieldIndexMap() { + return fieldIndexMap; + } + + public List getFieldList() { + return fieldList; + } + + public String getJoinedFieldNames() { + return tsColName + ", " + String.join(", ", fieldList); + } + + public int getTsColIndex() { + return tsColIndex; + } + + public String getTagValuesPlaceHolder() { + return tagList.stream().map(x -> "?").collect(Collectors.joining(",")); + } + + public String getFieldValuesPlaceHolder() { + return "?, " + fieldList.stream().map(x -> "?").collect(Collectors.joining(", ")); + } + + /** + * 计算子表表名 + *
    + *
  1. 将标签的value 组合成为如下的字符串: tag_value1!tag_value2!tag_value3。
    2. + *
  2. 计算该字符串的 MD5 散列值 "md5_val"。
    3. + *
  3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。
    4. + *
+ * + * @param tagValues + * @return + */ + public String computeTableName(String[] tagValues) { + String s = String.join("!", tagValues); + return "t_" + DigestUtils.md5Hex(s); + } + + public int getDynamicTagCount() { + return tagIndexMap.size(); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java new file mode 100644 index 0000000..79e5238 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -0,0 +1,91 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + + +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.spi.Writer; +import com.alibaba.datax.common.util.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Set; + +public class TDengineWriter extends Writer { + + private static final String PEER_PLUGIN_NAME = "peerPluginName"; + + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + + public static class Job extends Writer.Job { + + private Configuration originalConfig; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + this.originalConfig.set(PEER_PLUGIN_NAME, getPeerPluginName()); + } + + @Override + public void destroy() { + + } + + @Override + public List split(int mandatoryNumber) { + List writerSplitConfigs = new ArrayList<>(); + for (int i = 0; i < mandatoryNumber; i++) { + writerSplitConfigs.add(this.originalConfig); + } + return writerSplitConfigs; + } + } + + public static class Task extends Writer.Task { + private static final Logger LOG = LoggerFactory.getLogger(Job.class); + + private Configuration writerSliceConfig; + + @Override + public void init() { + this.writerSliceConfig = getPluginJobConf(); + + } + + @Override + public void destroy() { + + } + + @Override + public void startWrite(RecordReceiver lineReceiver) { + Set keys = this.writerSliceConfig.getKeys(); + Properties properties = new Properties(); + for (String key : keys) { + String value = this.writerSliceConfig.getString(key); + properties.setProperty(key, value); + } + if (!keys.contains(Key.USER)) { + properties.setProperty(Key.USER, "root"); + } + if (!keys.contains(Key.PASSWORD)) { + properties.setProperty(Key.PASSWORD, "taosdata"); + } + LOG.debug("========================properties==========================\n" + properties); + String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); + LOG.debug("start to handle record from: " + peerPluginName); + DataHandler handler = DataHandlerFactory.build(peerPluginName); + long records = handler.handle(lineReceiver, properties, getTaskPluginCollector()); + LOG.debug("handle data finished, records: " + records); + } + + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java new file mode 100644 index 0000000..994f1e8 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java @@ -0,0 +1,32 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum TDengineWriterErrorCode implements ErrorCode { + RUNTIME_EXCEPTION("TDengineWriter-00", "运行时异常"), + TYPE_ERROR("TDengineWriter-00", "Datax类型无法正确映射到TDengine类型"); + + private final String code; + private final String description; + + private TDengineWriterErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, + this.description); + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h new file mode 100644 index 0000000..4bdf363 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h @@ -0,0 +1,105 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_alibaba_datax_plugin_writer_JniConnection */ + +#ifndef _Included_com_alibaba_datax_plugin_writer_JniConnection +#define _Included_com_alibaba_datax_plugin_writer_JniConnection +#ifdef __cplusplus +extern "C" { +#endif +#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER +#define com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER 0LL +#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL +#define com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL 0L +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: initImp + * Signature: (Ljava/lang/String;)V + */ +JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_initImp + (JNIEnv *, jclass, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: setOptions + * Signature: (ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_setOptions + (JNIEnv *, jclass, jint, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getTsCharset + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getTsCharset + (JNIEnv *, jclass); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: connectImp + * Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_connectImp + (JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: executeQueryImp + * Signature: ([BJ)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_executeQueryImp + (JNIEnv *, jobject, jbyteArray, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrCodeImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrCodeImp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrMsgImp + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgImp + (JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getErrMsgByCode + * Signature: (J)Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgByCode + (JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: getAffectedRowsImp + * Signature: (JJ)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getAffectedRowsImp + (JNIEnv *, jobject, jlong, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: closeConnectionImp + * Signature: (J)I + */ +JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_closeConnectionImp + (JNIEnv *, jobject, jlong); + +/* + * Class: com_alibaba_datax_plugin_writer_JniConnection + * Method: insertOpentsdbJson + * Signature: (Ljava/lang/String;J)J + */ +JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_insertOpentsdbJson + (JNIEnv *, jobject, jstring, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/tdenginewriter/src/main/resources/plugin.json b/tdenginewriter/src/main/resources/plugin.json new file mode 100644 index 0000000..e54f65f --- /dev/null +++ b/tdenginewriter/src/main/resources/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "tdenginewriter", + "class": "com.alibaba.datax.plugin.writer.tdenginewriter.TDengineWriter", + "description": { + "useScene": "data migration to tdengine", + "mechanism": "use JNI or taos-jdbc to write data to tdengine." + }, + "developer": "zyyang-taosdata" +} \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/plugin_job_template.json b/tdenginewriter/src/main/resources/plugin_job_template.json new file mode 100644 index 0000000..5482b26 --- /dev/null +++ b/tdenginewriter/src/main/resources/plugin_job_template.json @@ -0,0 +1,10 @@ +{ + "name": "tdenginewriter", + "parameter": { + "host": "", + "port": 6030, + "db": "", + "user": "", + "password": "" + } +} \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg.properties b/tdenginewriter/src/main/resources/tdenginewritermsg.properties new file mode 100644 index 0000000..4aaa220 --- /dev/null +++ b/tdenginewriter/src/main/resources/tdenginewritermsg.properties @@ -0,0 +1,6 @@ +try_get_schema_fromdb=fail to get structure info of target table from configure file and will try to get it from database +batch_size_too_small='batchSize' is too small, please increase it and try again +column_number_error=number of columns is less than expected +tag_value_error=tag columns include 'null' value +ts_value_error=timestamp column type error or null +infer_column_type_error=fail to infer column type: sample count %d, column index %d \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties new file mode 100644 index 0000000..4aaa220 --- /dev/null +++ b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties @@ -0,0 +1,6 @@ +try_get_schema_fromdb=fail to get structure info of target table from configure file and will try to get it from database +batch_size_too_small='batchSize' is too small, please increase it and try again +column_number_error=number of columns is less than expected +tag_value_error=tag columns include 'null' value +ts_value_error=timestamp column type error or null +infer_column_type_error=fail to infer column type: sample count %d, column index %d \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties new file mode 100644 index 0000000..4b9552f --- /dev/null +++ b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties @@ -0,0 +1,6 @@ +try_get_schema_fromdb=\u65e0\u6cd5\u4ece\u914d\u7f6e\u6587\u4ef6\u83b7\u53d6\u8868\u7ed3\u6784\u4fe1\u606f\uff0c\u5c1d\u8bd5\u4ece\u6570\u636e\u5e93\u83b7\u53d6 +batch_size_too_small=batchSize\u592a\u5c0f\uff0c\u4f1a\u589e\u52a0\u81ea\u52a8\u7c7b\u578b\u63a8\u65ad\u9519\u8bef\u7684\u6982\u7387\uff0c\u5efa\u8bae\u6539\u5927\u540e\u91cd\u8bd5 +column_number_error=\u5b9e\u9645\u5217\u6570\u5c0f\u4e8e\u671f\u671b\u5217\u6570 +tag_value_error=\u6807\u7b7e\u5217\u5305\u542bnull +ts_value_error=\u65f6\u95f4\u6233\u5217\u4e3anull\u6216\u7c7b\u578b\u9519\u8bef +infer_column_type_error=\u6839\u636e\u91c7\u6837\u7684%d\u6761\u6570\u636e\uff0c\u65e0\u6cd5\u63a8\u65ad\u7b2c%d\u5217\u7684\u6570\u636e\u7c7b\u578b diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java new file mode 100644 index 0000000..09c3df2 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java @@ -0,0 +1,21 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import org.junit.Test; + +import java.util.Properties; + +public class JniConnectionTest { + + @Test + public void test() throws Exception { + JniConnection connection = new JniConnection(new Properties()); + + connection.open("192.168.56.105", 6030, "test", "root", "taosdata"); + + String json = "{\"metric\":\"weather_temperature\",\"timestamp\":1609430400000,\"value\":123,\"tags\":{\"location\":\"beijing\",\"id\":\"t123\"}}"; + connection.insertOpentsdbJson(json); + + connection.close(); + } + +} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java new file mode 100644 index 0000000..b1b7ddd --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java @@ -0,0 +1,25 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import org.junit.Test; + +import java.util.Locale; +import java.util.ResourceBundle; + +import org.junit.Assert; + +public class MessageTest { + @Test + public void testChineseMessage() { + Locale local = new Locale("zh", "CN"); + ResourceBundle bundle = ResourceBundle.getBundle("tdenginewritermsg", local); + String msg = bundle.getString("try_get_schema_fromdb"); + Assert.assertEquals("无法从配置文件获取表结构信息，尝试从数据库获取", msg); + } + + @Test + public void testDefaultMessage() { + ResourceBundle bundle = ResourceBundle.getBundle("tdenginewritermsg", Locale.getDefault()); + String msg = bundle.getString("try_get_schema_fromdb"); + System.out.println(msg); + } +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java new file mode 100644 index 0000000..62bf704 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java @@ -0,0 +1,31 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class TDengineWriterTest { + + + @Test + public void testGetSchema() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", "wozai.fun", "6030", "test", "root", "taosdata"); + Connection conn = DriverManager.getConnection(jdbcUrl); + SchemaManager schemaManager = new SchemaManager(); + schemaManager.setStable("test1"); + schemaManager.getFromDB(conn); + } + + @Test + public void dropTestTable() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", "wozai.fun", "6030", "test", "root", "taosdata"); + Connection conn = DriverManager.getConnection(jdbcUrl); + Statement stmt = conn.createStatement(); + stmt.execute("drop table market_snapshot"); + } +} From 72c72612fdacf731f662024f780febc547f1b384 Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 29 Nov 2021 13:37:47 +0800 Subject: [PATCH 053/114] fix job template error --- .../main/resources/plugin_job_template.json | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tdenginewriter/src/main/resources/plugin_job_template.json b/tdenginewriter/src/main/resources/plugin_job_template.json index 5482b26..39c9c96 100644 --- a/tdenginewriter/src/main/resources/plugin_job_template.json +++ b/tdenginewriter/src/main/resources/plugin_job_template.json @@ -1,10 +1,24 @@ { "name": "tdenginewriter", "parameter": { - "host": "", + "host": "127.0.0.1", "port": 6030, - "db": "", - "user": "", - "password": "" + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } } } \ No newline at end of file From 4ba65407140f9d5e086c2a80708c667d7dc09532 Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 29 Nov 2021 13:37:47 +0800 Subject: [PATCH 054/114] fix job template error --- .../main/resources/plugin_job_template.json | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tdenginewriter/src/main/resources/plugin_job_template.json b/tdenginewriter/src/main/resources/plugin_job_template.json index 5482b26..39c9c96 100644 --- a/tdenginewriter/src/main/resources/plugin_job_template.json +++ b/tdenginewriter/src/main/resources/plugin_job_template.json @@ -1,10 +1,24 @@ { "name": "tdenginewriter", "parameter": { - "host": "", + "host": "127.0.0.1", "port": 6030, - "db": "", - "user": "", - "password": "" + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } } } \ No newline at end of file From 400b34e8cd34542703ce18efd7b2a8de219f1a9a Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 29 Nov 2021 14:42:01 +0800 Subject: [PATCH 055/114] add pom module tdengine writer --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 3bd75a3..1a9da81 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,7 @@ mysqlwriter + tdenginewriter drdswriter odpswriter txtfilewriter From 30fa8f12843e478b70bd8ab64f9f4581b07d89f9 Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 29 Nov 2021 21:08:56 +0800 Subject: [PATCH 056/114] add English doc and fix bugs of Chinese doc --- tdenginewriter/doc/tdenginewriter-EN.md | 349 ++++++++++++++++++++++++ tdenginewriter/doc/tdenginewriter.md | 7 +- 2 files changed, 353 insertions(+), 3 deletions(-) create mode 100644 tdenginewriter/doc/tdenginewriter-EN.md diff --git a/tdenginewriter/doc/tdenginewriter-EN.md b/tdenginewriter/doc/tdenginewriter-EN.md new file mode 100644 index 0000000..eda88a9 --- /dev/null +++ b/tdenginewriter/doc/tdenginewriter-EN.md @@ -0,0 +1,349 @@ +# DataX TDengineWriter + +[简体中文](./tdenginewriter.md) | English + +## 1 Quick Introduction + +TDengineWriter Plugin writes data to [TDengine](https://www.taosdata.com/en/). It can be used to offline synchronize data from other databases to TDengine. + +## 2 Implementation + +TDengineWriter get records from DataX Framework that are generated from reader side. It has two whiting strategies: + +1. For data from OpenTSDBReader which is in json format, to leverage the new feature of TDengine Server that support writing json data directly called [schemaless writing](https://www.taosdata.com/cn/documentation/insert#schemaless), we use JNI to call functions in `taos.lib` or `taos.dll`.(Since the feature was not included in taos-jdbcdrive until version 2.0.36). +2. For other data sources, we use [taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java) to write data. If the target table is not exists beforehand, then it will be created automatically according to your configuration. + +## 3 Features Introduction +### 3.1 From OpenTSDB to TDengine +#### 3.1.1 Sample Setting + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather_temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.1.180", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +#### 3.1.2 Configuration + +| Parameter | Description | Required | Default | +| --------- | ------------------------------ | -------- | -------- | +| host | host of TDengine | Yes | | +| port | port of TDengine | Yes | | +| user | use name of TDengine | No | root | +| password | password of TDengine | No | taosdata | +| dbname | name of target database | No | | +| batchSize | batch size of insert operation | No | 1 | + + +#### 3.1.3 Type Convert + +| OpenTSDB Type | DataX Type | TDengine Type | +| ---------------- | ---------- | ------------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | + +### 3.2 From MongoDB to TDengine + +#### 3.2.1 Sample Setting +```json +{ + "job": { + "setting": { + "speed": { + "channel": 2 + } + }, + "content": [ + { + "reader": { + "name": "mongodbreader", + "parameter": { + "address": [ + "127.0.0.1:27017" + ], + "userName": "user", + "mechanism": "SCRAM-SHA-1", + "userPassword": "password", + "authDb": "admin", + "dbName": "test", + "collectionName": "stock", + "column": [ + { + "name": "stockID", + "type": "string" + }, + { + "name": "tradeTime", + "type": "date" + }, + { + "name": "lastPrice", + "type": "double" + }, + { + "name": "askPrice1", + "type": "double" + }, + { + "name": "bidPrice1", + "type": "double" + }, + { + "name": "volume", + "type": "int" + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "localhost", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "stable": "stock", + "tagColumn": { + "industry": "energy", + "stockID": 0 + }, + "fieldColumn": { + "lastPrice": 2, + "askPrice1": 3, + "bidPrice1": 4, + "volume": 5 + }, + "timestampColumn": { + "tradeTime": 1 + } + } + } + } + ] + } +} +``` + +**Note：the writer part of this setting can also apply to other data source except for OpenTSDB ** + + +#### 3.2.2 Configuration + +| Parameter | Description | Required | Default | Remark | +| --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | +| host | host ofTDengine | Yes | | +| port | port of TDengine | Yes | | +| user | user name of TDengine | No | root | +| password | password of TDengine | No | taosdata | +| dbname | name of target database | Yes | | +| batchSize | batch size of insert operation | No | 1000 | +| stable | name of target super table | Yes(except for OpenTSDB) | | +| tagColumn | name and position of tag columns in the record from reader | No | | index starts with 0 | +| fieldColumn | name and position of data columns in the record from reader | No | | | +| timestampColumn | name and position of timestamp column in the record from reader | No | | | + +#### 3.2.3 Auto table creating +##### 3.2.3.1 Rules + +If all of `tagColumn`, `fieldColumn` and `timestampColumn` are offered in writer configuration, then target super table will be created automatically. +The type of tag columns will always be `NCHAR(64)`. The sample setting above will produce following sql: + +```sql +CREATE STABLE IF NOT EXISTS market_snapshot ( + tadetime TIMESTAMP, + lastprice DOUBLE, + askprice1 DOUBLE, + bidprice1 DOUBLE, + volume INT +) +TAGS( + industry NCHAR(64), + stockID NCHAR(64) +); +``` + +##### 3.2.3.2 Sub-table Creating Rules + +The structure of sub-tables are the same with structure of super table. The names of sub-tables are generated by rules below: +1. combine value of tags like this:`tag_value1!tag_value2!tag_value3`. +2. compute md5 hash hex of above string, named `md5val` +3. use "t_md5val" as sub-table name, in which "t" is fixed prefix. + +#### 3.2.4 Use Pre-created Table + +If you have created super table firstly, then all of tagColumn, fieldColumn and timestampColumn can be omitted. The writer plugin will get table schema by executing `describe stableName`. +The order of columns of records received by this plugin must be the same as the order of columns returned by `describe stableName`. For example, if you have super table as below: +``` + Field | Type | Length | Note | +================================================================================= + ts | TIMESTAMP | 8 | | + current | DOUBLE | 8 | | + location | BINARY | 10 | TAG | +``` +Then the first columns received by this writer plugin must represent timestamp, the second column must represent current with type double, the third column must represent location with internal type string. + +#### 3.2.5 Remarks + +1. Config keys --tagColumn, fieldColumn and timestampColumn, must be presented or omitted at the same time. +2. If above three config keys exist and the target table also exists, then the order of columns defined by the config file and the existed table must be the same. + +#### 3.2.6 Type Convert + +| MongoDB Type | DataX Type | TDengine Type | +| ---------------- | -------------- | ----------------- | +| int, Long | Long | BIGINT | +| double | Double | DOUBLE | +| string, array | String | NCHAR(64) | +| date | Date | TIMESTAMP | +| boolean | Boolean | BOOL | +| bytes | Bytes | BINARY | + +### 3.3 From Relational Database to TDengine + +Take MySQl as example. + +#### 3.3.1 Table Structure in MySQL +```sql +CREATE TABLE IF NOT EXISTS weather( + station varchar(100), + latitude DOUBLE, + longtitude DOUBLE, + `date` DATE, + TMAX int, + TMIN int +) +``` + +#### 3.3.2 Sample Setting + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "passw0rd", + "column": [ + "*" + ], + "splitPk": "station", + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + + +## 4 Performance Test + +## 5 Restriction + +1. NCHAR type has fixed length 64 when auto creating stable. +2. Rows have null tag values will be dropped. + +## FAQ + +### How to filter on source table？ + +It depends on reader plugin. For different reader plugins, the way may be different. + +### How to import multiple source tables at once？ + +It depends on reader plugin. If the reader plugin supports reading multiple tables at once, then there is no problem. + +### How many sub-tables will be produced? + +The number of sub-tables is determined by tagColumns, equals to the number of different combinations of tag values. + +### Do columns in source table and columns in target table must be in the same order? + +No. TDengine require the first column has timestamp type，which is followed by data columns, followed by tag columns. The writer plugin will create super table in this column order, regardless of origin column orders. + +### How dose the plugin infer the data type of incoming data? + +By the first batch of records it received. + +### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. + +Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. \ No newline at end of file diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 062fac2..e318ef7 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -1,5 +1,7 @@ # DataX TDengineWriter +简体中文| [English](./tdenginewriter-EN.md) + ## 1 快速介绍 TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 @@ -203,7 +205,7 @@ TAGS( ##### 3.2.3.2 子表创建规则 -子表结果与超表相同，子表表名生成规则： +子表结构与超表相同，子表表名生成规则： 1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 2. 计算该字符串的 MD5 散列值 "md5_val"。 3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 @@ -225,7 +227,6 @@ TAGS( 1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 -3. 插件优先使用配置文件中指定的表结构。 #### 3.2.6 类型转换 @@ -387,7 +388,7 @@ CREATE TABLE IF NOT EXISTS weather( ### 源表和目标表的字段顺序一致吗？ -TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表是自动调整。 +TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表时会自动调整。 ### 插件如何确定各列的数据类型？ From e8563a23796be77c72587b5369f7eabe8dab9eb5 Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 30 Nov 2021 07:41:25 +0800 Subject: [PATCH 057/114] =?UTF-8?q?=E8=B6=85=E8=A1=A8=20to=20=E8=B6=85?= =?UTF-8?q?=E7=BA=A7=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tdenginewriter/doc/tdenginewriter.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index e318ef7..251f29a 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -205,7 +205,7 @@ TAGS( ##### 3.2.3.2 子表创建规则 -子表结构与超表相同，子表表名生成规则： +子表结构与超级表相同，子表表名生成规则： 1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 2. 计算该字符串的 MD5 散列值 "md5_val"。 3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 @@ -384,7 +384,7 @@ CREATE TABLE IF NOT EXISTS weather( ### 一张源表导入之后对应TDengine中多少张表？ -这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超表就有多少子表。 +这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 ### 源表和目标表的字段顺序一致吗？ From d554e26fa0654e28af0c29a4733bebb4a5b2ea1b Mon Sep 17 00:00:00 2001 From: dingbo Date: Tue, 30 Nov 2021 12:53:47 +0800 Subject: [PATCH 058/114] english doc as default --- tdenginewriter/doc/tdenginewriter-CN.md | 399 ++++++++++++++++++++++++ tdenginewriter/doc/tdenginewriter-EN.md | 349 --------------------- tdenginewriter/doc/tdenginewriter.md | 220 +++++-------- 3 files changed, 484 insertions(+), 484 deletions(-) create mode 100644 tdenginewriter/doc/tdenginewriter-CN.md delete mode 100644 tdenginewriter/doc/tdenginewriter-EN.md diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md new file mode 100644 index 0000000..c20b3b8 --- /dev/null +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -0,0 +1,399 @@ +# DataX TDengineWriter + +简体中文| [English](./tdenginewriter.md) + +## 1 快速介绍 + +TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 + +## 2 实现原理 + +TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： + +1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 + +2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 + +这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 + +## 3 功能说明 +### 3.1 从OpenTSDB到TDengine +#### 3.1.1 配置样例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather_temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.1.180", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +#### 3.1.2 参数说明 + +| 参数 | 描述 | 是否必选 | 默认值 | +| --------- | -------------------- | -------- | -------- | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbname | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1 | + + +#### 3.1.3 类型转换 + +目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： + +| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | + +### 3.2 从MongoDB到TDengine + +#### 3.2.1 配置样例 +```json +{ + "job": { + "setting": { + "speed": { + "channel": 2 + } + }, + "content": [ + { + "reader": { + "name": "mongodbreader", + "parameter": { + "address": [ + "127.0.0.1:27017" + ], + "userName": "user", + "mechanism": "SCRAM-SHA-1", + "userPassword": "password", + "authDb": "admin", + "dbName": "test", + "collectionName": "stock", + "column": [ + { + "name": "stockID", + "type": "string" + }, + { + "name": "tradeTime", + "type": "date" + }, + { + "name": "lastPrice", + "type": "double" + }, + { + "name": "askPrice1", + "type": "double" + }, + { + "name": "bidPrice1", + "type": "double" + }, + { + "name": "volume", + "type": "int" + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "localhost", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "stable": "stock", + "tagColumn": { + "industry": "energy", + "stockID": 0 + }, + "fieldColumn": { + "lastPrice": 2, + "askPrice1": 3, + "bidPrice1": 4, + "volume": 5 + }, + "timestampColumn": { + "tradeTime": 1 + } + } + } + } + ] + } +} +``` + +**注：本配置的writer部分同样适用于关系型数据库** + + +#### 3.2.2 参数说明 +| 参数 | 描述 | 是否必选 | 默认值 | 备注 | +| --------------- | -------------------- | ---------------- | -------- | ------------------ | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbname | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1000 | +| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | +| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | +| fieldColumn | 字段列的列名和位置 | 否 | 无 | | +| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | + +#### 3.2.3 自动建表规则 +##### 3.2.3.1 超级表创建规则 + +如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
+数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： + +```sql +CREATE STABLE IF NOT EXISTS market_snapshot ( + tadetime TIMESTAMP, + lastprice DOUBLE, + askprice1 DOUBLE, + bidprice1 DOUBLE, + volume INT +) +TAGS( + industry NCHAR(64), + stockID NCHAR(64) +); +``` + +##### 3.2.3.2 子表创建规则 + +子表结构与超级表相同，子表表名生成规则： +1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 +2. 计算该字符串的 MD5 散列值 "md5_val"。 +3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 + +#### 3.2.4 用户提前建表 + +如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 +此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： +``` + Field | Type | Length | Note | +================================================================================= + ts | TIMESTAMP | 8 | | + current | DOUBLE | 8 | | + location | BINARY | 10 | TAG | +``` +那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 + +#### 3.2.5 注意事项 + +1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 +2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 + +#### 3.2.6 类型转换 + +| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| int, Long | Long | BIGINT | +| double | Double | DOUBLE | +| string, array | String | NCHAR(64) | +| date | Date | TIMESTAMP | +| boolean | Boolean | BOOL | +| bytes | Bytes | BINARY | + +### 3.3 从关系型数据库到TDengine +writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 + +#### 3.3.1 MySQL中表结构 +```sql +CREATE TABLE IF NOT EXISTS weather( + station varchar(100), + latitude DOUBLE, + longtitude DOUBLE, + `date` DATE, + TMAX int, + TMIN int +) +``` + +#### 3.3.2 配置文件示例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "passw0rd", + "column": [ + "*" + ], + "splitPk": "station", + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "dbname": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数 | DataX速度(Rec/s) | DataX流量(MB/s) | DataX机器网卡流出流量(MB/s) | DataX机器运行负载 | DB网卡进入流量(MB/s) | DB运行负载 | DB TPS | +| ------ | ---------------- | --------------- | --------------------------- | ----------------- | -------------------- | ---------- | ------ | +| 1 | | | | | | | | +| 4 | | | | | | | | +| 8 | | | | | | | | +| 16 | | | | | | | | +| 32 | | | | | | | | + +说明： + +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 +3. 16通道，4096批量提交时，出现 full gc 2次。 + +#### 4.2.4 性能测试小结 + + +## 5 约束限制 + +1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 +2. 标签列不能包含null值，如果包含会被过滤掉。 + +## FAQ + +### 如何选取要同步的数据的范围？ + +数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 + +### 如何一次导入多张源表？ + +如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 + +### 一张源表导入之后对应TDengine中多少张表？ + +这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 + +### 源表和目标表的字段顺序一致吗？ + +TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表时会自动调整。 + +### 插件如何确定各列的数据类型？ + +根据收到的第一批数据自动推断各列的类型。 + +### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? + +因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file diff --git a/tdenginewriter/doc/tdenginewriter-EN.md b/tdenginewriter/doc/tdenginewriter-EN.md deleted file mode 100644 index eda88a9..0000000 --- a/tdenginewriter/doc/tdenginewriter-EN.md +++ /dev/null @@ -1,349 +0,0 @@ -# DataX TDengineWriter - -[简体中文](./tdenginewriter.md) | English - -## 1 Quick Introduction - -TDengineWriter Plugin writes data to [TDengine](https://www.taosdata.com/en/). It can be used to offline synchronize data from other databases to TDengine. - -## 2 Implementation - -TDengineWriter get records from DataX Framework that are generated from reader side. It has two whiting strategies: - -1. For data from OpenTSDBReader which is in json format, to leverage the new feature of TDengine Server that support writing json data directly called [schemaless writing](https://www.taosdata.com/cn/documentation/insert#schemaless), we use JNI to call functions in `taos.lib` or `taos.dll`.(Since the feature was not included in taos-jdbcdrive until version 2.0.36). -2. For other data sources, we use [taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java) to write data. If the target table is not exists beforehand, then it will be created automatically according to your configuration. - -## 3 Features Introduction -### 3.1 From OpenTSDB to TDengine -#### 3.1.1 Sample Setting - -```json -{ - "job": { - "content": [ - { - "reader": { - "name": "opentsdbreader", - "parameter": { - "endpoint": "http://192.168.1.180:4242", - "column": [ - "weather_temperature" - ], - "beginDateTime": "2021-01-01 00:00:00", - "endDateTime": "2021-01-01 01:00:00" - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "192.168.1.180", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata" - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} -``` - -#### 3.1.2 Configuration - -| Parameter | Description | Required | Default | -| --------- | ------------------------------ | -------- | -------- | -| host | host of TDengine | Yes | | -| port | port of TDengine | Yes | | -| user | use name of TDengine | No | root | -| password | password of TDengine | No | taosdata | -| dbname | name of target database | No | | -| batchSize | batch size of insert operation | No | 1 | - - -#### 3.1.3 Type Convert - -| OpenTSDB Type | DataX Type | TDengine Type | -| ---------------- | ---------- | ------------- | -| timestamp | Date | timestamp | -| Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | -| Integer（tag） | String | binary | -| Float（tag） | String | binary | -| String（tag） | String | binary | - -### 3.2 From MongoDB to TDengine - -#### 3.2.1 Sample Setting -```json -{ - "job": { - "setting": { - "speed": { - "channel": 2 - } - }, - "content": [ - { - "reader": { - "name": "mongodbreader", - "parameter": { - "address": [ - "127.0.0.1:27017" - ], - "userName": "user", - "mechanism": "SCRAM-SHA-1", - "userPassword": "password", - "authDb": "admin", - "dbName": "test", - "collectionName": "stock", - "column": [ - { - "name": "stockID", - "type": "string" - }, - { - "name": "tradeTime", - "type": "date" - }, - { - "name": "lastPrice", - "type": "double" - }, - { - "name": "askPrice1", - "type": "double" - }, - { - "name": "bidPrice1", - "type": "double" - }, - { - "name": "volume", - "type": "int" - } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "localhost", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata", - "stable": "stock", - "tagColumn": { - "industry": "energy", - "stockID": 0 - }, - "fieldColumn": { - "lastPrice": 2, - "askPrice1": 3, - "bidPrice1": 4, - "volume": 5 - }, - "timestampColumn": { - "tradeTime": 1 - } - } - } - } - ] - } -} -``` - -**Note：the writer part of this setting can also apply to other data source except for OpenTSDB ** - - -#### 3.2.2 Configuration - -| Parameter | Description | Required | Default | Remark | -| --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | -| host | host ofTDengine | Yes | | -| port | port of TDengine | Yes | | -| user | user name of TDengine | No | root | -| password | password of TDengine | No | taosdata | -| dbname | name of target database | Yes | | -| batchSize | batch size of insert operation | No | 1000 | -| stable | name of target super table | Yes(except for OpenTSDB) | | -| tagColumn | name and position of tag columns in the record from reader | No | | index starts with 0 | -| fieldColumn | name and position of data columns in the record from reader | No | | | -| timestampColumn | name and position of timestamp column in the record from reader | No | | | - -#### 3.2.3 Auto table creating -##### 3.2.3.1 Rules - -If all of `tagColumn`, `fieldColumn` and `timestampColumn` are offered in writer configuration, then target super table will be created automatically. -The type of tag columns will always be `NCHAR(64)`. The sample setting above will produce following sql: - -```sql -CREATE STABLE IF NOT EXISTS market_snapshot ( - tadetime TIMESTAMP, - lastprice DOUBLE, - askprice1 DOUBLE, - bidprice1 DOUBLE, - volume INT -) -TAGS( - industry NCHAR(64), - stockID NCHAR(64) -); -``` - -##### 3.2.3.2 Sub-table Creating Rules - -The structure of sub-tables are the same with structure of super table. The names of sub-tables are generated by rules below: -1. combine value of tags like this:`tag_value1!tag_value2!tag_value3`. -2. compute md5 hash hex of above string, named `md5val` -3. use "t_md5val" as sub-table name, in which "t" is fixed prefix. - -#### 3.2.4 Use Pre-created Table - -If you have created super table firstly, then all of tagColumn, fieldColumn and timestampColumn can be omitted. The writer plugin will get table schema by executing `describe stableName`. -The order of columns of records received by this plugin must be the same as the order of columns returned by `describe stableName`. For example, if you have super table as below: -``` - Field | Type | Length | Note | -================================================================================= - ts | TIMESTAMP | 8 | | - current | DOUBLE | 8 | | - location | BINARY | 10 | TAG | -``` -Then the first columns received by this writer plugin must represent timestamp, the second column must represent current with type double, the third column must represent location with internal type string. - -#### 3.2.5 Remarks - -1. Config keys --tagColumn, fieldColumn and timestampColumn, must be presented or omitted at the same time. -2. If above three config keys exist and the target table also exists, then the order of columns defined by the config file and the existed table must be the same. - -#### 3.2.6 Type Convert - -| MongoDB Type | DataX Type | TDengine Type | -| ---------------- | -------------- | ----------------- | -| int, Long | Long | BIGINT | -| double | Double | DOUBLE | -| string, array | String | NCHAR(64) | -| date | Date | TIMESTAMP | -| boolean | Boolean | BOOL | -| bytes | Bytes | BINARY | - -### 3.3 From Relational Database to TDengine - -Take MySQl as example. - -#### 3.3.1 Table Structure in MySQL -```sql -CREATE TABLE IF NOT EXISTS weather( - station varchar(100), - latitude DOUBLE, - longtitude DOUBLE, - `date` DATE, - TMAX int, - TMIN int -) -``` - -#### 3.3.2 Sample Setting - -```json -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "username": "root", - "password": "passw0rd", - "column": [ - "*" - ], - "splitPk": "station", - "connection": [ - { - "table": [ - "weather" - ], - "jdbcUrl": [ - "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" - ] - } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "127.0.0.1", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata", - "batchSize": 1000, - "stable": "weather", - "tagColumn": { - "station": 0 - }, - "fieldColumn": { - "latitude": 1, - "longtitude": 2, - "tmax": 4, - "tmin": 5 - }, - "timestampColumn":{ - "date": 3 - } - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} -``` - - -## 4 Performance Test - -## 5 Restriction - -1. NCHAR type has fixed length 64 when auto creating stable. -2. Rows have null tag values will be dropped. - -## FAQ - -### How to filter on source table？ - -It depends on reader plugin. For different reader plugins, the way may be different. - -### How to import multiple source tables at once？ - -It depends on reader plugin. If the reader plugin supports reading multiple tables at once, then there is no problem. - -### How many sub-tables will be produced? - -The number of sub-tables is determined by tagColumns, equals to the number of different combinations of tag values. - -### Do columns in source table and columns in target table must be in the same order? - -No. TDengine require the first column has timestamp type，which is followed by data columns, followed by tag columns. The writer plugin will create super table in this column order, regardless of origin column orders. - -### How dose the plugin infer the data type of incoming data? - -By the first batch of records it received. - -### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. - -Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. \ No newline at end of file diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 251f29a..42bdd04 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -1,24 +1,21 @@ # DataX TDengineWriter -简体中文| [English](./tdenginewriter-EN.md) +[简体中文](./tdenginewriter-CN.md) | English -## 1 快速介绍 +## 1 Quick Introduction -TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 +TDengineWriter Plugin writes data to [TDengine](https://www.taosdata.com/en/). It can be used to offline synchronize data from other databases to TDengine. -## 2 实现原理 +## 2 Implementation -TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： +TDengineWriter get records from DataX Framework that are generated from reader side. It has two whiting strategies: -1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 +1. For data from OpenTSDBReader which is in json format, to leverage the new feature of TDengine Server that support writing json data directly called [schemaless writing](https://www.taosdata.com/cn/documentation/insert#schemaless), we use JNI to call functions in `taos.lib` or `taos.dll`.(Since the feature was not included in taos-jdbcdrive until version 2.0.36). +2. For other data sources, we use [taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java) to write data. If the target table is not exists beforehand, then it will be created automatically according to your configuration. -2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 - -这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 - -## 3 功能说明 -### 3.1 从OpenTSDB到TDengine -#### 3.1.1 配置样例 +## 3 Features Introduction +### 3.1 From OpenTSDB to TDengine +#### 3.1.1 Sample Setting ```json { @@ -57,35 +54,33 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea } ``` -#### 3.1.2 参数说明 +#### 3.1.2 Configuration -| 参数 | 描述 | 是否必选 | 默认值 | -| --------- | -------------------- | -------- | -------- | -| host | TDengine实例的host | 是 | 无 | -| port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | -| password | TDengine实例的密码 | 否 | taosdata | -| dbname | 目的数据库的名称 | 是 | 无 | -| batchSize | 每次批量插入多少记录 | 否 | 1 | +| Parameter | Description | Required | Default | +| --------- | ------------------------------ | -------- | -------- | +| host | host of TDengine | Yes | | +| port | port of TDengine | Yes | | +| user | use name of TDengine | No | root | +| password | password of TDengine | No | taosdata | +| dbname | name of target database | No | | +| batchSize | batch size of insert operation | No | 1 | -#### 3.1.3 类型转换 +#### 3.1.3 Type Convert -目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： +| OpenTSDB Type | DataX Type | TDengine Type | +| ---------------- | ---------- | ------------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | -| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | -| ---------------- | -------------- | ----------------- | -| timestamp | Date | timestamp | -| Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | -| Integer（tag） | String | binary | -| Float（tag） | String | binary | -| String（tag） | String | binary | +### 3.2 From MongoDB to TDengine -### 3.2 从MongoDB到TDengine - -#### 3.2.1 配置样例 +#### 3.2.1 Sample Setting ```json { "job": { @@ -166,28 +161,29 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea } ``` -**注：本配置的writer部分同样适用于关系型数据库** +**Note：the writer part of this setting can also apply to other data source except for OpenTSDB ** -#### 3.2.2 参数说明 -| 参数 | 描述 | 是否必选 | 默认值 | 备注 | -| --------------- | -------------------- | ---------------- | -------- | ------------------ | -| host | TDengine实例的host | 是 | 无 | -| port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | -| password | TDengine实例的密码 | 否 | taosdata | -| dbname | 目的数据库的名称 | 是 | 无 | -| batchSize | 每次批量插入多少记录 | 否 | 1000 | -| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | -| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | -| fieldColumn | 字段列的列名和位置 | 否 | 无 | | -| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | +#### 3.2.2 Configuration -#### 3.2.3 自动建表规则 -##### 3.2.3.1 超级表创建规则 +| Parameter | Description | Required | Default | Remark | +| --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | +| host | host ofTDengine | Yes | | +| port | port of TDengine | Yes | | +| user | user name of TDengine | No | root | +| password | password of TDengine | No | taosdata | +| dbname | name of target database | Yes | | +| batchSize | batch size of insert operation | No | 1000 | +| stable | name of target super table | Yes(except for OpenTSDB) | | +| tagColumn | name and position of tag columns in the record from reader | No | | index starts with 0 | +| fieldColumn | name and position of data columns in the record from reader | No | | | +| timestampColumn | name and position of timestamp column in the record from reader | No | | | -如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
-数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： +#### 3.2.3 Auto table creating +##### 3.2.3.1 Rules + +If all of `tagColumn`, `fieldColumn` and `timestampColumn` are offered in writer configuration, then target super table will be created automatically. +The type of tag columns will always be `NCHAR(64)`. The sample setting above will produce following sql: ```sql CREATE STABLE IF NOT EXISTS market_snapshot ( @@ -203,17 +199,17 @@ TAGS( ); ``` -##### 3.2.3.2 子表创建规则 +##### 3.2.3.2 Sub-table Creating Rules -子表结构与超级表相同，子表表名生成规则： -1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 -2. 计算该字符串的 MD5 散列值 "md5_val"。 -3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 +The structure of sub-tables are the same with structure of super table. The names of sub-tables are generated by rules below: +1. combine value of tags like this:`tag_value1!tag_value2!tag_value3`. +2. compute md5 hash hex of above string, named `md5val` +3. use "t_md5val" as sub-table name, in which "t" is fixed prefix. -#### 3.2.4 用户提前建表 +#### 3.2.4 Use Pre-created Table -如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 -此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： +If you have created super table firstly, then all of tagColumn, fieldColumn and timestampColumn can be omitted. The writer plugin will get table schema by executing `describe stableName`. +The order of columns of records received by this plugin must be the same as the order of columns returned by `describe stableName`. For example, if you have super table as below: ``` Field | Type | Length | Note | ================================================================================= @@ -221,16 +217,16 @@ TAGS( current | DOUBLE | 8 | | location | BINARY | 10 | TAG | ``` -那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 +Then the first columns received by this writer plugin must represent timestamp, the second column must represent current with type double, the third column must represent location with internal type string. -#### 3.2.5 注意事项 +#### 3.2.5 Remarks -1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 -2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 +1. Config keys --tagColumn, fieldColumn and timestampColumn, must be presented or omitted at the same time. +2. If above three config keys exist and the target table also exists, then the order of columns defined by the config file and the existed table must be the same. -#### 3.2.6 类型转换 +#### 3.2.6 Type Convert -| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | +| MongoDB Type | DataX Type | TDengine Type | | ---------------- | -------------- | ----------------- | | int, Long | Long | BIGINT | | double | Double | DOUBLE | @@ -239,10 +235,11 @@ TAGS( | boolean | Boolean | BOOL | | bytes | Bytes | BINARY | -### 3.3 从关系型数据库到TDengine -writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 +### 3.3 From Relational Database to TDengine -#### 3.3.1 MySQL中表结构 +Take MySQl as example. + +#### 3.3.1 Table Structure in MySQL ```sql CREATE TABLE IF NOT EXISTS weather( station varchar(100), @@ -254,7 +251,7 @@ CREATE TABLE IF NOT EXISTS weather( ) ``` -#### 3.3.2 配置文件示例 +#### 3.3.2 Sample Setting ```json { @@ -318,82 +315,35 @@ CREATE TABLE IF NOT EXISTS weather( ``` -## 4 性能报告 +## 4 Performance Test -### 4.1 环境准备 +## 5 Restriction -#### 4.1.1 数据特征 - -建表语句： - -单行记录类似于： - -#### 4.1.2 机器参数 - -* 执行DataX的机器参数为: - 1. cpu: - 2. mem: - 3. net: 千兆双网卡 - 4. disc: DataX 数据不落磁盘，不统计此项 - -* TDengine数据库机器参数为: - 1. cpu: - 2. mem: - 3. net: 千兆双网卡 - 4. disc: - -#### 4.1.3 DataX jvm 参数 - - -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError - -### 4.2 测试报告 - -#### 4.2.1 单表测试报告 - -| 通道数 | DataX速度(Rec/s) | DataX流量(MB/s) | DataX机器网卡流出流量(MB/s) | DataX机器运行负载 | DB网卡进入流量(MB/s) | DB运行负载 | DB TPS | -| ------ | ---------------- | --------------- | --------------------------- | ----------------- | -------------------- | ---------- | ------ | -| 1 | | | | | | | | -| 4 | | | | | | | | -| 8 | | | | | | | | -| 16 | | | | | | | | -| 32 | | | | | | | | - -说明： - -1. 这里的单表，主键类型为 bigint(20),自增。 -2. batchSize 和 通道个数，对性能影响较大。 -3. 16通道，4096批量提交时，出现 full gc 2次。 - -#### 4.2.4 性能测试小结 - - -## 5 约束限制 - -1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 -2. 标签列不能包含null值，如果包含会被过滤掉。 +1. NCHAR type has fixed length 64 when auto creating stable. +2. Rows have null tag values will be dropped. ## FAQ -### 如何选取要同步的数据的范围？ +### How to filter on source table？ -数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 +It depends on reader plugin. For different reader plugins, the way may be different. -### 如何一次导入多张源表？ +### How to import multiple source tables at once？ -如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 +It depends on reader plugin. If the reader plugin supports reading multiple tables at once, then there is no problem. -### 一张源表导入之后对应TDengine中多少张表？ +### How many sub-tables will be produced? -这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 +The number of sub-tables is determined by tagColumns, equals to the number of different combinations of tag values. -### 源表和目标表的字段顺序一致吗？ +### Do columns in source table and columns in target table must be in the same order? -TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表时会自动调整。 +No. TDengine require the first column has timestamp type，which is followed by data columns, followed by tag columns. The writer plugin will create super table in this column order, regardless of origin column orders. -### 插件如何确定各列的数据类型？ +### How dose the plugin infer the data type of incoming data? -根据收到的第一批数据自动推断各列的类型。 +By the first batch of records it received. -### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? +### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. -因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file +Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. \ No newline at end of file From beb0d76136246c1d462cdba3214b309c7fb31282 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 2 Dec 2021 15:08:52 +0800 Subject: [PATCH 059/114] rename dbname to dbName. use english doc as default. --- tdenginewriter/doc/tdenginewriter-CN.md | 399 ++++++++++++++++++ tdenginewriter/doc/tdenginewriter.md | 244 ++++------- .../plugin/writer/tdenginewriter/Key.java | 2 +- 3 files changed, 489 insertions(+), 156 deletions(-) create mode 100644 tdenginewriter/doc/tdenginewriter-CN.md diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md new file mode 100644 index 0000000..ffd3efc --- /dev/null +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -0,0 +1,399 @@ +# DataX TDengineWriter + +简体中文| [English](./tdenginewriter.md) + +## 1 快速介绍 + +TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 + +## 2 实现原理 + +TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： + +1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 + +2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 + +这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 + +## 3 功能说明 +### 3.1 从OpenTSDB到TDengine +#### 3.1.1 配置样例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": [ + "weather_temperature" + ], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.1.180", + "port": 6030, + "dbName": "test", + "user": "root", + "password": "taosdata" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +#### 3.1.2 参数说明 + +| 参数 | 描述 | 是否必选 | 默认值 | +| --------- | -------------------- | -------- | -------- | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbName | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1 | + + +#### 3.1.3 类型转换 + +目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： + +| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | + +### 3.2 从MongoDB到TDengine + +#### 3.2.1 配置样例 +```json +{ + "job": { + "setting": { + "speed": { + "channel": 2 + } + }, + "content": [ + { + "reader": { + "name": "mongodbreader", + "parameter": { + "address": [ + "127.0.0.1:27017" + ], + "userName": "user", + "mechanism": "SCRAM-SHA-1", + "userPassword": "password", + "authDb": "admin", + "dbName": "test", + "collectionName": "stock", + "column": [ + { + "name": "stockID", + "type": "string" + }, + { + "name": "tradeTime", + "type": "date" + }, + { + "name": "lastPrice", + "type": "double" + }, + { + "name": "askPrice1", + "type": "double" + }, + { + "name": "bidPrice1", + "type": "double" + }, + { + "name": "volume", + "type": "int" + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "localhost", + "port": 6030, + "dbName": "test", + "user": "root", + "password": "taosdata", + "stable": "stock", + "tagColumn": { + "industry": "energy", + "stockID": 0 + }, + "fieldColumn": { + "lastPrice": 2, + "askPrice1": 3, + "bidPrice1": 4, + "volume": 5 + }, + "timestampColumn": { + "tradeTime": 1 + } + } + } + } + ] + } +} +``` + +**注：本配置的writer部分同样适用于关系型数据库** + + +#### 3.2.2 参数说明 +| 参数 | 描述 | 是否必选 | 默认值 | 备注 | +| --------------- | -------------------- | ---------------- | -------- | ------------------ | +| host | TDengine实例的host | 是 | 无 | +| port | TDengine实例的port | 是 | 无 | +| user | TDengine实例的用户名 | 否 | root | +| password | TDengine实例的密码 | 否 | taosdata | +| dbName | 目的数据库的名称 | 是 | 无 | +| batchSize | 每次批量插入多少记录 | 否 | 1000 | +| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | +| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | +| fieldColumn | 字段列的列名和位置 | 否 | 无 | | +| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | + +#### 3.2.3 自动建表规则 +##### 3.2.3.1 超级表创建规则 + +如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
+数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： + +```sql +CREATE STABLE IF NOT EXISTS market_snapshot ( + tadetime TIMESTAMP, + lastprice DOUBLE, + askprice1 DOUBLE, + bidprice1 DOUBLE, + volume INT +) +TAGS( + industry NCHAR(64), + stockID NCHAR(64) +); +``` + +##### 3.2.3.2 子表创建规则 + +子表结构与超级表相同，子表表名生成规则： +1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 +2. 计算该字符串的 MD5 散列值 "md5_val"。 +3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 + +#### 3.2.4 用户提前建表 + +如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 +此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： +``` + Field | Type | Length | Note | +================================================================================= + ts | TIMESTAMP | 8 | | + current | DOUBLE | 8 | | + location | BINARY | 10 | TAG | +``` +那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 + +#### 3.2.5 注意事项 + +1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 +2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 + +#### 3.2.6 类型转换 + +| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | +| ---------------- | -------------- | ----------------- | +| int, Long | Long | BIGINT | +| double | Double | DOUBLE | +| string, array | String | NCHAR(64) | +| date | Date | TIMESTAMP | +| boolean | Boolean | BOOL | +| bytes | Bytes | BINARY(64) | + +### 3.3 从关系型数据库到TDengine +writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 + +#### 3.3.1 MySQL中表结构 +```sql +CREATE TABLE IF NOT EXISTS weather( + station varchar(100), + latitude DOUBLE, + longtitude DOUBLE, + `date` DATE, + TMAX int, + TMIN int +) +``` + +#### 3.3.2 配置文件示例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "passw0rd", + "column": [ + "*" + ], + "splitPk": "station", + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": [ + "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "dbName": "test", + "user": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "weather", + "tagColumn": { + "station": 0 + }, + "fieldColumn": { + "latitude": 1, + "longtitude": 2, + "tmax": 4, + "tmin": 5 + }, + "timestampColumn":{ + "date": 3 + } + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数 | DataX速度(Rec/s) | DataX流量(MB/s) | DataX机器网卡流出流量(MB/s) | DataX机器运行负载 | DB网卡进入流量(MB/s) | DB运行负载 | DB TPS | +| ------ | ---------------- | --------------- | --------------------------- | ----------------- | -------------------- | ---------- | ------ | +| 1 | | | | | | | | +| 4 | | | | | | | | +| 8 | | | | | | | | +| 16 | | | | | | | | +| 32 | | | | | | | | + +说明： + +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 +3. 16通道，4096批量提交时，出现 full gc 2次。 + +#### 4.2.4 性能测试小结 + + +## 5 约束限制 + +1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 +2. 标签列不能包含null值，如果包含会被过滤掉。 + +## FAQ + +### 如何选取要同步的数据的范围？ + +数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 + +### 如何一次导入多张源表？ + +如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 + +### 一张源表导入之后对应TDengine中多少张表？ + +这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 + +### 源表和目标表的字段顺序一致吗？ + +TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表时会自动调整。 + +### 插件如何确定各列的数据类型？ + +根据收到的第一批数据自动推断各列的类型。 + +### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? + +因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 3e8a5f7..022bd21 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -1,24 +1,21 @@ # DataX TDengineWriter -简体中文| [English](./tdenginewriter-EN.md) +[简体中文](./tdenginewriter-CN.md) | English -## 1 快速介绍 +## 1 Quick Introduction -TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 +TDengineWriter Plugin writes data to [TDengine](https://www.taosdata.com/en/). It can be used to offline synchronize data from other databases to TDengine. -## 2 实现原理 +## 2 Implementation -TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： +TDengineWriter get records from DataX Framework that are generated from reader side. It has two whiting strategies: -1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 +1. For data from OpenTSDBReader which is in json format, to leverage the new feature of TDengine Server that support writing json data directly called [schemaless writing](https://www.taosdata.com/cn/documentation/insert#schemaless), we use JNI to call functions in `taos.lib` or `taos.dll`.(Since the feature was not included in taos-jdbcdrive until version 2.0.36). +2. For other data sources, we use [taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java) to write data. If the target table is not exists beforehand, then it will be created automatically according to your configuration. -2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 - -这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 - -## 3 功能说明 -### 3.1 从OpenTSDB到TDengine -#### 3.1.1 配置样例 +## 3 Features Introduction +### 3.1 From OpenTSDB to TDengine +#### 3.1.1 Sample Setting ```json { @@ -41,7 +38,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "parameter": { "host": "192.168.1.180", "port": 6030, - "dbname": "test", + "dbName": "test", "user": "root", "password": "taosdata" } @@ -57,35 +54,33 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea } ``` -#### 3.1.2 参数说明 +#### 3.1.2 Configuration -| 参数 | 描述 | 是否必选 | 默认值 | -| --------- | -------------------- | -------- | -------- | -| host | TDengine实例的host | 是 | 无 | -| port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | -| password | TDengine实例的密码 | 否 | taosdata | -| dbname | 目的数据库的名称 | 是 | 无 | -| batchSize | 每次批量插入多少记录 | 否 | 1 | +| Parameter | Description | Required | Default | +| --------- | ------------------------------ | -------- | -------- | +| host | host of TDengine | Yes | | +| port | port of TDengine | Yes | | +| user | use name of TDengine | No | root | +| password | password of TDengine | No | taosdata | +| dbName | name of target database | No | | +| batchSize | batch size of insert operation | No | 1 | -#### 3.1.3 类型转换 +#### 3.1.3 Type Convert -目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： +| OpenTSDB Type | DataX Type | TDengine Type | +| ---------------- | ---------- | ------------- | +| timestamp | Date | timestamp | +| Integer（value） | Double | double | +| Float（value） | Double | double | +| String（value） | String | binary | +| Integer（tag） | String | binary | +| Float（tag） | String | binary | +| String（tag） | String | binary | -| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | -| ---------------- | -------------- | ----------------- | -| timestamp | Date | timestamp | -| Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | -| Integer（tag） | String | binary | -| Float（tag） | String | binary | -| String（tag） | String | binary | +### 3.2 From MongoDB to TDengine -### 3.2 从MongoDB到TDengine - -#### 3.2.1 配置样例 +#### 3.2.1 Sample Setting ```json { "job": { @@ -141,7 +136,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "parameter": { "host": "localhost", "port": 6030, - "dbname": "test", + "dbName": "test", "user": "root", "password": "taosdata", "stable": "stock", @@ -166,28 +161,29 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea } ``` -**注：本配置的writer部分同样适用于关系型数据库** +**Note：the writer part of this setting can also apply to other data source except for OpenTSDB ** -#### 3.2.2 参数说明 -| 参数 | 描述 | 是否必选 | 默认值 | 备注 | -| --------------- | -------------------- | ---------------- | -------- | ------------------ | -| host | TDengine实例的host | 是 | 无 | -| port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | -| password | TDengine实例的密码 | 否 | taosdata | -| dbname | 目的数据库的名称 | 是 | 无 | -| batchSize | 每次批量插入多少记录 | 否 | 1000 | -| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | -| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | -| fieldColumn | 字段列的列名和位置 | 否 | 无 | | -| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | +#### 3.2.2 Configuration -#### 3.2.3 自动建表规则 -##### 3.2.3.1 超级表创建规则 +| Parameter | Description | Required | Default | Remark | +| --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | +| host | host ofTDengine | Yes | | +| port | port of TDengine | Yes | | +| user | user name of TDengine | No | root | +| password | password of TDengine | No | taosdata | +| dbName | name of target database | Yes | | +| batchSize | batch size of insert operation | No | 1000 | +| stable | name of target super table | Yes(except for OpenTSDB) | | +| tagColumn | name and position of tag columns in the record from reader | No | | index starts with 0 | +| fieldColumn | name and position of data columns in the record from reader | No | | | +| timestampColumn | name and position of timestamp column in the record from reader | No | | | -如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
-数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： +#### 3.2.3 Auto table creating +##### 3.2.3.1 Rules + +If all of `tagColumn`, `fieldColumn` and `timestampColumn` are offered in writer configuration, then target super table will be created automatically. +The type of tag columns will always be `NCHAR(64)`. The sample setting above will produce following sql: ```sql CREATE STABLE IF NOT EXISTS market_snapshot ( @@ -203,21 +199,17 @@ TAGS( ); ``` -##### 3.2.3.2 子表创建规则 +##### 3.2.3.2 Sub-table Creating Rules -<<<<<<< HEAD -子表结果与超表相同，子表表名生成规则： -======= -子表结构与超级表相同，子表表名生成规则： ->>>>>>> TD-11503/english-doc-for-writer -1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 -2. 计算该字符串的 MD5 散列值 "md5_val"。 -3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 +The structure of sub-tables are the same with structure of super table. The names of sub-tables are generated by rules below: +1. combine value of tags like this:`tag_value1!tag_value2!tag_value3`. +2. compute md5 hash hex of above string, named `md5val` +3. use "t_md5val" as sub-table name, in which "t" is fixed prefix. -#### 3.2.4 用户提前建表 +#### 3.2.4 Use Pre-created Table -如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 -此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： +If you have created super table firstly, then all of tagColumn, fieldColumn and timestampColumn can be omitted. The writer plugin will get table schema by executing `describe stableName`. +The order of columns of records received by this plugin must be the same as the order of columns returned by `describe stableName`. For example, if you have super table as below: ``` Field | Type | Length | Note | ================================================================================= @@ -225,32 +217,29 @@ TAGS( current | DOUBLE | 8 | | location | BINARY | 10 | TAG | ``` -那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 +Then the first columns received by this writer plugin must represent timestamp, the second column must represent current with type double, the third column must represent location with internal type string. -#### 3.2.5 注意事项 +#### 3.2.5 Remarks -1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 -2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 -<<<<<<< HEAD -3. 插件优先使用配置文件中指定的表结构。 -======= ->>>>>>> TD-11503/english-doc-for-writer +1. Config keys --tagColumn, fieldColumn and timestampColumn, must be presented or omitted at the same time. +2. If above three config keys exist and the target table also exists, then the order of columns defined by the config file and the existed table must be the same. -#### 3.2.6 类型转换 +#### 3.2.6 Type Convert -| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | +| MongoDB Type | DataX Type | TDengine Type | | ---------------- | -------------- | ----------------- | | int, Long | Long | BIGINT | | double | Double | DOUBLE | | string, array | String | NCHAR(64) | | date | Date | TIMESTAMP | | boolean | Boolean | BOOL | -| bytes | Bytes | BINARY | +| bytes | Bytes | BINARY(64) | -### 3.3 从关系型数据库到TDengine -writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 +### 3.3 From Relational Database to TDengine -#### 3.3.1 MySQL中表结构 +Take MySQl as example. + +#### 3.3.1 Table Structure in MySQL ```sql CREATE TABLE IF NOT EXISTS weather( station varchar(100), @@ -262,7 +251,7 @@ CREATE TABLE IF NOT EXISTS weather( ) ``` -#### 3.3.2 配置文件示例 +#### 3.3.2 Sample Setting ```json { @@ -295,7 +284,7 @@ CREATE TABLE IF NOT EXISTS weather( "parameter": { "host": "127.0.0.1", "port": 6030, - "dbname": "test", + "dbName": "test", "user": "root", "password": "taosdata", "batchSize": 1000, @@ -326,90 +315,35 @@ CREATE TABLE IF NOT EXISTS weather( ``` -## 4 性能报告 +## 4 Performance Test -### 4.1 环境准备 +## 5 Restriction -#### 4.1.1 数据特征 - -建表语句： - -单行记录类似于： - -#### 4.1.2 机器参数 - -* 执行DataX的机器参数为: - 1. cpu: - 2. mem: - 3. net: 千兆双网卡 - 4. disc: DataX 数据不落磁盘，不统计此项 - -* TDengine数据库机器参数为: - 1. cpu: - 2. mem: - 3. net: 千兆双网卡 - 4. disc: - -#### 4.1.3 DataX jvm 参数 - - -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError - -### 4.2 测试报告 - -#### 4.2.1 单表测试报告 - -| 通道数 | DataX速度(Rec/s) | DataX流量(MB/s) | DataX机器网卡流出流量(MB/s) | DataX机器运行负载 | DB网卡进入流量(MB/s) | DB运行负载 | DB TPS | -| ------ | ---------------- | --------------- | --------------------------- | ----------------- | -------------------- | ---------- | ------ | -| 1 | | | | | | | | -| 4 | | | | | | | | -| 8 | | | | | | | | -| 16 | | | | | | | | -| 32 | | | | | | | | - -说明： - -1. 这里的单表，主键类型为 bigint(20),自增。 -2. batchSize 和 通道个数，对性能影响较大。 -3. 16通道，4096批量提交时，出现 full gc 2次。 - -#### 4.2.4 性能测试小结 - - -## 5 约束限制 - -1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 -2. 标签列不能包含null值，如果包含会被过滤掉。 +1. NCHAR type has fixed length 64 when auto creating stable. +2. Rows have null tag values will be dropped. ## FAQ -### 如何选取要同步的数据的范围？ +### How to filter on source table？ -数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 +It depends on reader plugin. For different reader plugins, the way may be different. -### 如何一次导入多张源表？ +### How to import multiple source tables at once？ -如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 +It depends on reader plugin. If the reader plugin supports reading multiple tables at once, then there is no problem. -### 一张源表导入之后对应TDengine中多少张表？ +### How many sub-tables will be produced? -<<<<<<< HEAD -这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超表就有多少子表。 +The number of sub-tables is determined by tagColumns, equals to the number of different combinations of tag values. -### 源表和目标表的字段顺序一致吗？ +### Do columns in source table and columns in target table must be in the same order? -TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表是自动调整。 -======= -这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 +No. TDengine require the first column has timestamp type，which is followed by data columns, followed by tag columns. The writer plugin will create super table in this column order, regardless of origin column orders. -### 源表和目标表的字段顺序一致吗？ +### How dose the plugin infer the data type of incoming data? -TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表时会自动调整。 ->>>>>>> TD-11503/english-doc-for-writer +By the first batch of records it received. -### 插件如何确定各列的数据类型？ +### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. -根据收到的第一批数据自动推断各列的类型。 - -### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? - -因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file +Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index 090a799..a061e97 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -3,7 +3,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; public class Key { public static final String HOST = "host"; public static final String PORT = "port"; - public static final String DBNAME = "dbname"; + public static final String DBNAME = "dbName"; public static final String USER = "user"; public static final String PASSWORD = "password"; public static final String BATCH_SIZE = "batchSize"; From 5d58600e44160dedf68f550330147951f219a1a7 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 3 Dec 2021 16:04:51 +0800 Subject: [PATCH 060/114] add tdenginewriter to package.xml --- package.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/package.xml b/package.xml index 882dd23..d95feb4 100755 --- a/package.xml +++ b/package.xml @@ -189,6 +189,13 @@ datax + + tdenginewriter/target/datax/ + + **/*.* + + datax + drdswriter/target/datax/ From 1372a653cf40f5ca6d04c7ff050878af04bd9815 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 3 Dec 2021 16:34:19 +0800 Subject: [PATCH 061/114] delete duplicate en doc --- tdenginewriter/doc/tdenginewriter-EN.md | 349 ------------------------ 1 file changed, 349 deletions(-) delete mode 100644 tdenginewriter/doc/tdenginewriter-EN.md diff --git a/tdenginewriter/doc/tdenginewriter-EN.md b/tdenginewriter/doc/tdenginewriter-EN.md deleted file mode 100644 index eda88a9..0000000 --- a/tdenginewriter/doc/tdenginewriter-EN.md +++ /dev/null @@ -1,349 +0,0 @@ -# DataX TDengineWriter - -[简体中文](./tdenginewriter.md) | English - -## 1 Quick Introduction - -TDengineWriter Plugin writes data to [TDengine](https://www.taosdata.com/en/). It can be used to offline synchronize data from other databases to TDengine. - -## 2 Implementation - -TDengineWriter get records from DataX Framework that are generated from reader side. It has two whiting strategies: - -1. For data from OpenTSDBReader which is in json format, to leverage the new feature of TDengine Server that support writing json data directly called [schemaless writing](https://www.taosdata.com/cn/documentation/insert#schemaless), we use JNI to call functions in `taos.lib` or `taos.dll`.(Since the feature was not included in taos-jdbcdrive until version 2.0.36). -2. For other data sources, we use [taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java) to write data. If the target table is not exists beforehand, then it will be created automatically according to your configuration. - -## 3 Features Introduction -### 3.1 From OpenTSDB to TDengine -#### 3.1.1 Sample Setting - -```json -{ - "job": { - "content": [ - { - "reader": { - "name": "opentsdbreader", - "parameter": { - "endpoint": "http://192.168.1.180:4242", - "column": [ - "weather_temperature" - ], - "beginDateTime": "2021-01-01 00:00:00", - "endDateTime": "2021-01-01 01:00:00" - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "192.168.1.180", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata" - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} -``` - -#### 3.1.2 Configuration - -| Parameter | Description | Required | Default | -| --------- | ------------------------------ | -------- | -------- | -| host | host of TDengine | Yes | | -| port | port of TDengine | Yes | | -| user | use name of TDengine | No | root | -| password | password of TDengine | No | taosdata | -| dbname | name of target database | No | | -| batchSize | batch size of insert operation | No | 1 | - - -#### 3.1.3 Type Convert - -| OpenTSDB Type | DataX Type | TDengine Type | -| ---------------- | ---------- | ------------- | -| timestamp | Date | timestamp | -| Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | -| Integer（tag） | String | binary | -| Float（tag） | String | binary | -| String（tag） | String | binary | - -### 3.2 From MongoDB to TDengine - -#### 3.2.1 Sample Setting -```json -{ - "job": { - "setting": { - "speed": { - "channel": 2 - } - }, - "content": [ - { - "reader": { - "name": "mongodbreader", - "parameter": { - "address": [ - "127.0.0.1:27017" - ], - "userName": "user", - "mechanism": "SCRAM-SHA-1", - "userPassword": "password", - "authDb": "admin", - "dbName": "test", - "collectionName": "stock", - "column": [ - { - "name": "stockID", - "type": "string" - }, - { - "name": "tradeTime", - "type": "date" - }, - { - "name": "lastPrice", - "type": "double" - }, - { - "name": "askPrice1", - "type": "double" - }, - { - "name": "bidPrice1", - "type": "double" - }, - { - "name": "volume", - "type": "int" - } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "localhost", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata", - "stable": "stock", - "tagColumn": { - "industry": "energy", - "stockID": 0 - }, - "fieldColumn": { - "lastPrice": 2, - "askPrice1": 3, - "bidPrice1": 4, - "volume": 5 - }, - "timestampColumn": { - "tradeTime": 1 - } - } - } - } - ] - } -} -``` - -**Note：the writer part of this setting can also apply to other data source except for OpenTSDB ** - - -#### 3.2.2 Configuration - -| Parameter | Description | Required | Default | Remark | -| --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | -| host | host ofTDengine | Yes | | -| port | port of TDengine | Yes | | -| user | user name of TDengine | No | root | -| password | password of TDengine | No | taosdata | -| dbname | name of target database | Yes | | -| batchSize | batch size of insert operation | No | 1000 | -| stable | name of target super table | Yes(except for OpenTSDB) | | -| tagColumn | name and position of tag columns in the record from reader | No | | index starts with 0 | -| fieldColumn | name and position of data columns in the record from reader | No | | | -| timestampColumn | name and position of timestamp column in the record from reader | No | | | - -#### 3.2.3 Auto table creating -##### 3.2.3.1 Rules - -If all of `tagColumn`, `fieldColumn` and `timestampColumn` are offered in writer configuration, then target super table will be created automatically. -The type of tag columns will always be `NCHAR(64)`. The sample setting above will produce following sql: - -```sql -CREATE STABLE IF NOT EXISTS market_snapshot ( - tadetime TIMESTAMP, - lastprice DOUBLE, - askprice1 DOUBLE, - bidprice1 DOUBLE, - volume INT -) -TAGS( - industry NCHAR(64), - stockID NCHAR(64) -); -``` - -##### 3.2.3.2 Sub-table Creating Rules - -The structure of sub-tables are the same with structure of super table. The names of sub-tables are generated by rules below: -1. combine value of tags like this:`tag_value1!tag_value2!tag_value3`. -2. compute md5 hash hex of above string, named `md5val` -3. use "t_md5val" as sub-table name, in which "t" is fixed prefix. - -#### 3.2.4 Use Pre-created Table - -If you have created super table firstly, then all of tagColumn, fieldColumn and timestampColumn can be omitted. The writer plugin will get table schema by executing `describe stableName`. -The order of columns of records received by this plugin must be the same as the order of columns returned by `describe stableName`. For example, if you have super table as below: -``` - Field | Type | Length | Note | -================================================================================= - ts | TIMESTAMP | 8 | | - current | DOUBLE | 8 | | - location | BINARY | 10 | TAG | -``` -Then the first columns received by this writer plugin must represent timestamp, the second column must represent current with type double, the third column must represent location with internal type string. - -#### 3.2.5 Remarks - -1. Config keys --tagColumn, fieldColumn and timestampColumn, must be presented or omitted at the same time. -2. If above three config keys exist and the target table also exists, then the order of columns defined by the config file and the existed table must be the same. - -#### 3.2.6 Type Convert - -| MongoDB Type | DataX Type | TDengine Type | -| ---------------- | -------------- | ----------------- | -| int, Long | Long | BIGINT | -| double | Double | DOUBLE | -| string, array | String | NCHAR(64) | -| date | Date | TIMESTAMP | -| boolean | Boolean | BOOL | -| bytes | Bytes | BINARY | - -### 3.3 From Relational Database to TDengine - -Take MySQl as example. - -#### 3.3.1 Table Structure in MySQL -```sql -CREATE TABLE IF NOT EXISTS weather( - station varchar(100), - latitude DOUBLE, - longtitude DOUBLE, - `date` DATE, - TMAX int, - TMIN int -) -``` - -#### 3.3.2 Sample Setting - -```json -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "username": "root", - "password": "passw0rd", - "column": [ - "*" - ], - "splitPk": "station", - "connection": [ - { - "table": [ - "weather" - ], - "jdbcUrl": [ - "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" - ] - } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "127.0.0.1", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata", - "batchSize": 1000, - "stable": "weather", - "tagColumn": { - "station": 0 - }, - "fieldColumn": { - "latitude": 1, - "longtitude": 2, - "tmax": 4, - "tmin": 5 - }, - "timestampColumn":{ - "date": 3 - } - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} -``` - - -## 4 Performance Test - -## 5 Restriction - -1. NCHAR type has fixed length 64 when auto creating stable. -2. Rows have null tag values will be dropped. - -## FAQ - -### How to filter on source table？ - -It depends on reader plugin. For different reader plugins, the way may be different. - -### How to import multiple source tables at once？ - -It depends on reader plugin. If the reader plugin supports reading multiple tables at once, then there is no problem. - -### How many sub-tables will be produced? - -The number of sub-tables is determined by tagColumns, equals to the number of different combinations of tag values. - -### Do columns in source table and columns in target table must be in the same order? - -No. TDengine require the first column has timestamp type，which is followed by data columns, followed by tag columns. The writer plugin will create super table in this column order, regardless of origin column orders. - -### How dose the plugin infer the data type of incoming data? - -By the first batch of records it received. - -### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. - -Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. \ No newline at end of file From 75c032c4f93c3662ac1870819e6836888a8ec846 Mon Sep 17 00:00:00 2001 From: dingbo Date: Fri, 3 Dec 2021 17:09:50 +0800 Subject: [PATCH 062/114] refine doc --- tdenginewriter/doc/tdenginewriter-CN.md | 14 ++++++++++---- tdenginewriter/doc/tdenginewriter.md | 13 ++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md index ffd3efc..abd92de 100644 --- a/tdenginewriter/doc/tdenginewriter-CN.md +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -179,9 +179,11 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea | dbName | 目的数据库的名称 | 是 | 无 | | batchSize | 每次批量插入多少记录 | 否 | 1000 | | stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | -| tagColumn | 标签列的列名和位置 | 否 | 无 | 位置索引均从0开始 | -| fieldColumn | 字段列的列名和位置 | 否 | 无 | | -| timestampColumn | 时间戳列的列名和位置 | 否 | 无 | 时间戳列只能有一个 | +| tagColumn | 格式:{tagName1: tagInd1, tagName2: tagInd2}, 标签列在写插件收到的Record中的位置和列名 | 否 | 无 | 位置索引均从0开始, tagInd如果为字符串, 表示固定标签值，不需要从源数据中获取 | +| fieldColumn | 格式:{fdName1: fdInd1, fdName2: fdInd2}, 字段列在写插件收到的Record中的位置和列名 | 否 | 无 | | +| timestampColumn | 格式:{tsColName: tsColIndex}, 时间戳列在写插件收到的Record中的位置和列名 | 否 | 无 | 时间戳列只能有一个 | + +示例配置中tagColumn有一个industry，它的值是一个固定的字符串“energy”, 作用是给导入的所有数据加一个值为"energy"的固定标签industry。这个应用场景可以是：在源库中，有多个设备采集的数据分表存储，设备名就是表名，可以用这个机制把设备名称转化为标签。 #### 3.2.3 自动建表规则 ##### 3.2.3.1 超级表创建规则 @@ -396,4 +398,8 @@ TDengine要求每个表第一列是时间戳列，后边是普通字段，最后 ### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? -因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 \ No newline at end of file +因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 + +### 如果编译的时候某些插件的依赖找不到怎么办？ + +如果这个插件不是必须的，可以注释掉根目录下的pom.xml中的对应插件。 \ No newline at end of file diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 022bd21..fd19057 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -175,10 +175,12 @@ TDengineWriter get records from DataX Framework that are generated from reader s | dbName | name of target database | Yes | | | batchSize | batch size of insert operation | No | 1000 | | stable | name of target super table | Yes(except for OpenTSDB) | | -| tagColumn | name and position of tag columns in the record from reader | No | | index starts with 0 | -| fieldColumn | name and position of data columns in the record from reader | No | | | +| tagColumn | name and position of tag columns in the record from reader, format:{tagName1: tagInd1, tagName2: tagInd2} | No | | index starts with 0 | +| fieldColumn | name and position of data columns in the record from reader, format: {fdName1: fdInd1, fdName2: fdInd2} | No | | | | timestampColumn | name and position of timestamp column in the record from reader | No | | | +**Note**: You see that the value of tagColumn "industry" is a fixed string, this ia a good feature of this plugin. Think about this scenario: you have many tables with the structure and one table corresponds to one device. You want to use the device number as a tag in the target super table, then this feature is designed for you. + #### 3.2.3 Auto table creating ##### 3.2.3.1 Rules @@ -346,4 +348,9 @@ By the first batch of records it received. ### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. -Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. \ No newline at end of file +Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. + + +### What should I do if some dependencies of a plugin can't be found? + +I this plugin is not necessary for you, just remove it from pom.xml under project's root directory. From 2fa114838d5b68a73dd1be1782c241229db2f24a Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 6 Dec 2021 14:59:10 +0800 Subject: [PATCH 063/114] remove mongodb type --- tdenginewriter/doc/tdenginewriter-CN.md | 16 ++++++++-------- tdenginewriter/doc/tdenginewriter.md | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md index abd92de..c0ce768 100644 --- a/tdenginewriter/doc/tdenginewriter-CN.md +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -232,14 +232,14 @@ TAGS( #### 3.2.6 类型转换 -| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | -| ---------------- | -------------- | ----------------- | -| int, Long | Long | BIGINT | -| double | Double | DOUBLE | -| string, array | String | NCHAR(64) | -| date | Date | TIMESTAMP | -| boolean | Boolean | BOOL | -| bytes | Bytes | BINARY(64) | +| DataX 内部类型 | TDengine 数据类型 | +|-------------- | ----------------- | +|Long | BIGINT | +|Double | DOUBLE | +|String | NCHAR(64) | +|Date | TIMESTAMP | +|Boolean | BOOL | +|Bytes | BINARY(64) | ### 3.3 从关系型数据库到TDengine writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index fd19057..1e7fb64 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -228,14 +228,14 @@ Then the first columns received by this writer plugin must represent timestamp, #### 3.2.6 Type Convert -| MongoDB Type | DataX Type | TDengine Type | -| ---------------- | -------------- | ----------------- | -| int, Long | Long | BIGINT | -| double | Double | DOUBLE | -| string, array | String | NCHAR(64) | -| date | Date | TIMESTAMP | -| boolean | Boolean | BOOL | -| bytes | Bytes | BINARY(64) | +|DataX Type | TDengine Type | +|-------------- | ----------------- | +|Long | BIGINT | +|Double | DOUBLE | +|String | NCHAR(64) | +|Date | TIMESTAMP | +|Boolean | BOOL | +|Bytes | BINARY(64) | ### 3.3 From Relational Database to TDengine From e4bcc0e50e045711ff1dab3f423bfa4030244ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91=E6=98=9F?= Date: Thu, 18 Nov 2021 16:55:02 +0800 Subject: [PATCH 064/114] feat: tsdb2tsdb [mput/username/password/database] --- package.xml | 7 + .../plugin/reader/tsdbreader/Constant.java | 2 + .../datax/plugin/reader/tsdbreader/Key.java | 5 + .../plugin/reader/tsdbreader/TSDBReader.java | 109 ++++-- .../tsdbreader/conn/Connection4TSDB.java | 27 +- .../tsdbreader/conn/TSDBConnection.java | 41 ++- .../reader/tsdbreader/conn/TSDBDump.java | 96 +++++- .../reader/tsdbreader/util/HttpUtils.java | 50 ++- .../reader/tsdbreader/util/TSDBUtils.java | 38 +- tsdbwriter/pom.xml | 7 + .../plugin/writer/conn/Connection4TSDB.java | 34 +- .../plugin/writer/conn/TSDBConnection.java | 38 +- .../datax/plugin/writer/tsdbwriter/Key.java | 16 +- .../writer/tsdbwriter/SourceDBType.java | 5 + .../writer/tsdbwriter/TSDBConverter.java | 96 ++++++ .../plugin/writer/tsdbwriter/TSDBModel.java | 11 + .../plugin/writer/tsdbwriter/TSDBWriter.java | 324 +++++++++++++++--- .../tsdbwriter/TSDBWriterErrorCode.java | 1 + .../datax/plugin/writer/util/HttpUtils.java | 56 ++- .../datax/plugin/writer/util/TSDBUtils.java | 38 +- 20 files changed, 805 insertions(+), 196 deletions(-) create mode 100644 tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/SourceDBType.java create mode 100644 tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBConverter.java create mode 100644 tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBModel.java diff --git a/package.xml b/package.xml index 49e3c4e..3d86172 100755 --- a/package.xml +++ b/package.xml @@ -357,6 +357,13 @@ datax + + tsdbreader/target/datax/ + + **/*.* + + datax + adbpgwriter/target/datax/ diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java index e42dedc..f5069dc 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java @@ -16,6 +16,8 @@ public final class Constant { static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static final String METRIC_SPECIFY_KEY = "__metric__"; + public static final String METRIC_SPECIFY_KEY_PREFIX = METRIC_SPECIFY_KEY + "."; + public static final int METRIC_SPECIFY_KEY_PREFIX_LENGTH = METRIC_SPECIFY_KEY_PREFIX.length(); public static final String TS_SPECIFY_KEY = "__ts__"; public static final String VALUE_SPECIFY_KEY = "__value__"; diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java index 14ee7e4..c8a3d7a 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Key.java @@ -17,14 +17,19 @@ public class Key { // RDB for MySQL / ADB etc. static final String SINK_DB_TYPE = "sinkDbType"; static final String ENDPOINT = "endpoint"; + static final String USERNAME = "username"; + static final String PASSWORD = "password"; static final String COLUMN = "column"; static final String METRIC = "metric"; static final String FIELD = "field"; static final String TAG = "tag"; + static final String COMBINE = "combine"; static final String INTERVAL_DATE_TIME = "splitIntervalMs"; static final String BEGIN_DATE_TIME = "beginDateTime"; static final String END_DATE_TIME = "endDateTime"; + static final String HINT = "hint"; + static final Boolean COMBINE_DEFAULT_VALUE = false; static final Integer INTERVAL_DATE_TIME_DEFAULT_VALUE = 60; static final String TYPE_DEFAULT_VALUE = "TSDB"; static final Set TYPE_SET = new HashSet<>(); diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java index 04b931c..550a010 100755 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReader.java @@ -60,6 +60,15 @@ public class TSDBReader extends Reader { "The parameter [" + Key.ENDPOINT + "] is not set."); } + String username = originalConfig.getString(Key.USERNAME, null); + if (StringUtils.isBlank(username)) { + LOG.warn("The parameter [" + Key.USERNAME + "] is blank."); + } + String password = originalConfig.getString(Key.PASSWORD, null); + if (StringUtils.isBlank(password)) { + LOG.warn("The parameter [" + Key.PASSWORD + "] is blank."); + } + // tagK / field could be empty if ("TSDB".equals(type)) { List columns = originalConfig.getList(Key.COLUMN, String.class); @@ -76,7 +85,14 @@ public class TSDBReader extends Reader { "The parameter [" + Key.COLUMN + "] is not set."); } for (String specifyKey : Constant.MUST_CONTAINED_SPECIFY_KEYS) { - if (!columns.contains(specifyKey)) { + boolean containSpecifyKey = false; + for (String column : columns) { + if (column.startsWith(specifyKey)) { + containSpecifyKey = true; + break; + } + } + if (!containSpecifyKey) { throw DataXException.asDataXException( TSDBReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.COLUMN + "] should contain " @@ -99,6 +115,8 @@ public class TSDBReader extends Reader { "The parameter [" + Key.INTERVAL_DATE_TIME + "] should be great than zero."); } + Boolean isCombine = originalConfig.getBool(Key.COMBINE, Key.COMBINE_DEFAULT_VALUE); + SimpleDateFormat format = new SimpleDateFormat(Constant.DEFAULT_DATA_FORMAT); String startTime = originalConfig.getString(Key.BEGIN_DATE_TIME); Long startDate; @@ -168,14 +186,14 @@ public class TSDBReader extends Reader { startTime = format.parse(originalConfig.getString(Key.BEGIN_DATE_TIME)).getTime(); } catch (ParseException e) { throw DataXException.asDataXException( - TSDBReaderErrorCode.ILLEGAL_VALUE, "解析[" + Key.BEGIN_DATE_TIME + "]失败.", e); + TSDBReaderErrorCode.ILLEGAL_VALUE, "Analysis [" + Key.BEGIN_DATE_TIME + "] failed.", e); } long endTime; try { endTime = format.parse(originalConfig.getString(Key.END_DATE_TIME)).getTime(); } catch (ParseException e) { throw DataXException.asDataXException( - TSDBReaderErrorCode.ILLEGAL_VALUE, "解析[" + Key.END_DATE_TIME + "]失败.", e); + TSDBReaderErrorCode.ILLEGAL_VALUE, "Analysis [" + Key.END_DATE_TIME + "] failed.", e); } if (TimeUtils.isSecond(startTime)) { startTime *= 1000; @@ -186,13 +204,14 @@ public class TSDBReader extends Reader { DateTime startDateTime = new DateTime(TimeUtils.getTimeInHour(startTime)); DateTime endDateTime = new DateTime(TimeUtils.getTimeInHour(endTime)); + final Boolean isCombine = originalConfig.getBool(Key.COMBINE, Key.COMBINE_DEFAULT_VALUE); + if ("TSDB".equals(type)) { - // split by metric - for (String column : columns4TSDB) { + if (isCombine) { // split by time in hour while (startDateTime.isBefore(endDateTime)) { Configuration clone = this.originalConfig.clone(); - clone.set(Key.COLUMN, Collections.singletonList(column)); + clone.set(Key.COLUMN, columns4TSDB); clone.set(Key.BEGIN_DATE_TIME, startDateTime.getMillis()); startDateTime = startDateTime.plusMillis(splitIntervalMs); @@ -202,15 +221,30 @@ public class TSDBReader extends Reader { LOG.info("Configuration: {}", JSON.toJSONString(clone)); } + } else { + // split by time in hour + while (startDateTime.isBefore(endDateTime)) { + // split by metric + for (String column : columns4TSDB) { + Configuration clone = this.originalConfig.clone(); + clone.set(Key.COLUMN, Collections.singletonList(column)); + + clone.set(Key.BEGIN_DATE_TIME, startDateTime.getMillis()); + startDateTime = startDateTime.plusMillis(splitIntervalMs); + // Make sure the time interval is [start, end). + clone.set(Key.END_DATE_TIME, startDateTime.getMillis() - 1); + configurations.add(clone); + + LOG.info("Configuration: {}", JSON.toJSONString(clone)); + } + } } } else { - // split by metric - for (String metric : metrics) { - // split by time in hour + if (isCombine) { while (startDateTime.isBefore(endDateTime)) { Configuration clone = this.originalConfig.clone(); clone.set(Key.COLUMN, columns4RDB); - clone.set(Key.METRIC, Collections.singletonList(metric)); + clone.set(Key.METRIC, metrics); clone.set(Key.BEGIN_DATE_TIME, startDateTime.getMillis()); startDateTime = startDateTime.plusMillis(splitIntervalMs); @@ -220,6 +254,24 @@ public class TSDBReader extends Reader { LOG.info("Configuration: {}", JSON.toJSONString(clone)); } + } else { + // split by time in hour + while (startDateTime.isBefore(endDateTime)) { + // split by metric + for (String metric : metrics) { + Configuration clone = this.originalConfig.clone(); + clone.set(Key.COLUMN, columns4RDB); + clone.set(Key.METRIC, Collections.singletonList(metric)); + + clone.set(Key.BEGIN_DATE_TIME, startDateTime.getMillis()); + startDateTime = startDateTime.plusMillis(splitIntervalMs); + // Make sure the time interval is [start, end). + clone.set(Key.END_DATE_TIME, startDateTime.getMillis() - 1); + configurations.add(clone); + + LOG.info("Configuration: {}", JSON.toJSONString(clone)); + } + } } } return configurations; @@ -247,6 +299,8 @@ public class TSDBReader extends Reader { private TSDBConnection conn; private Long startTime; private Long endTime; + private Boolean isCombine; + private Map hint; @Override public void init() { @@ -265,11 +319,16 @@ public class TSDBReader extends Reader { this.tags = readerSliceConfig.getMap(Key.TAG); String address = readerSliceConfig.getString(Key.ENDPOINT); + String username = readerSliceConfig.getString(Key.USERNAME); + String password = readerSliceConfig.getString(Key.PASSWORD); - conn = new TSDBConnection(address); + conn = new TSDBConnection(address, username, password); this.startTime = readerSliceConfig.getLong(Key.BEGIN_DATE_TIME); this.endTime = readerSliceConfig.getLong(Key.END_DATE_TIME); + + this.isCombine = readerSliceConfig.getBool(Key.COMBINE, Key.COMBINE_DEFAULT_VALUE); + this.hint = readerSliceConfig.getMap(Key.HINT); } @Override @@ -283,29 +342,35 @@ public class TSDBReader extends Reader { if ("TSDB".equals(type)) { for (String metric : columns4TSDB) { final Map tags = this.tags == null ? - null : (Map) this.tags.get(metric); + null : (Map) this.tags.get(metric); if (fields == null || !fields.containsKey(metric)) { - conn.sendDPs(metric, tags, this.startTime, this.endTime, recordSender); + conn.sendDPs(metric, tags, this.startTime, this.endTime, recordSender, hint); } else { conn.sendDPs(metric, (List) fields.get(metric), - tags, this.startTime, this.endTime, recordSender); + tags, this.startTime, this.endTime, recordSender, hint); } } } else { - for (String metric : metrics) { + if (isCombine) { final Map tags = this.tags == null ? - null : (Map) this.tags.get(metric); - if (fields == null || !fields.containsKey(metric)) { - conn.sendRecords(metric, tags, startTime, endTime, columns4RDB, recordSender); - } else { - conn.sendRecords(metric, (List) fields.get(metric), - tags, startTime, endTime, columns4RDB, recordSender); + null : (Map) this.tags.get(metrics.get(0)); + conn.sendRecords(metrics, tags, startTime, endTime, columns4RDB, recordSender, hint); + } else { + for (String metric : metrics) { + final Map tags = this.tags == null ? + null : (Map) this.tags.get(metric); + if (fields == null || !fields.containsKey(metric)) { + conn.sendRecords(metric, tags, startTime, endTime, columns4RDB, isCombine, recordSender, hint); + } else { + conn.sendRecords(metric, (List) fields.get(metric), + tags, startTime, endTime, columns4RDB, recordSender, hint); + } } } } } catch (Exception e) { throw DataXException.asDataXException( - TSDBReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + TSDBReaderErrorCode.ILLEGAL_VALUE, "Error in getting or sending data point！", e); } } diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java index 500894b..96cb7f9 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/Connection4TSDB.java @@ -22,6 +22,20 @@ public interface Connection4TSDB { */ String address(); + /** + * Get the address of Database. + * + * @return host+ip + */ + String username(); + + /** + * Get the address of Database. + * + * @return host+ip + */ + String password(); + /** * Get the version of Database. * @@ -46,22 +60,27 @@ public interface Connection4TSDB { /** * Send data points for TSDB with single field. */ - void sendDPs(String metric, Map tags, Long start, Long end, RecordSender recordSender) throws Exception; + void sendDPs(String metric, Map tags, Long start, Long end, RecordSender recordSender, Map hint) throws Exception; /** * Send data points for TSDB with multi fields. */ - void sendDPs(String metric, List fields, Map tags, Long start, Long end, RecordSender recordSender) throws Exception; + void sendDPs(String metric, List fields, Map tags, Long start, Long end, RecordSender recordSender, Map hint) throws Exception; /** * Send data points for RDB with single field. */ - void sendRecords(String metric, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception; + void sendRecords(String metric, Map tags, Long start, Long end, List columns4RDB, Boolean isCombine, RecordSender recordSender, Map hint) throws Exception; /** * Send data points for RDB with multi fields. */ - void sendRecords(String metric, List fields, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception; + void sendRecords(String metric, List fields, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender, Map hint) throws Exception; + + /** + * Send data points for RDB with single fields on combine mode. + */ + void sendRecords(List metrics, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender, Map hint) throws Exception; /** * Put data point. diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java index 5426ab4..d466da3 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnection.java @@ -19,9 +19,13 @@ import java.util.Map; public class TSDBConnection implements Connection4TSDB { private String address; + private String username; + private String password; - public TSDBConnection(String address) { + public TSDBConnection(String address, String username, String password) { this.address = address; + this.username = username; + this.password = password; } @Override @@ -29,14 +33,24 @@ public class TSDBConnection implements Connection4TSDB { return address; } + @Override + public String username() { + return username; + } + + @Override + public String password() { + return password; + } + @Override public String version() { - return TSDBUtils.version(address); + return TSDBUtils.version(address, username, password); } @Override public String config() { - return TSDBUtils.config(address); + return TSDBUtils.config(address, username, password); } @Override @@ -45,23 +59,28 @@ public class TSDBConnection implements Connection4TSDB { } @Override - public void sendDPs(String metric, Map tags, Long start, Long end, RecordSender recordSender) throws Exception { - TSDBDump.dump4TSDB(this, metric, tags, start, end, recordSender); + public void sendDPs(String metric, Map tags, Long start, Long end, RecordSender recordSender, Map hint) throws Exception { + TSDBDump.dump4TSDB(this, metric, tags, start, end, recordSender, hint); } @Override - public void sendDPs(String metric, List fields, Map tags, Long start, Long end, RecordSender recordSender) throws Exception { - TSDBDump.dump4TSDB(this, metric, fields, tags, start, end, recordSender); + public void sendDPs(String metric, List fields, Map tags, Long start, Long end, RecordSender recordSender, Map hint) throws Exception { + TSDBDump.dump4TSDB(this, metric, fields, tags, start, end, recordSender, hint); } @Override - public void sendRecords(String metric, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception { - TSDBDump.dump4RDB(this, metric, tags, start, end, columns4RDB, recordSender); + public void sendRecords(String metric, Map tags, Long start, Long end, List columns4RDB, Boolean isCombine, RecordSender recordSender, Map hint) throws Exception { + TSDBDump.dump4RDB(this, metric, tags, start, end, columns4RDB, recordSender, hint); } @Override - public void sendRecords(String metric, List fields, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender) throws Exception { - TSDBDump.dump4RDB(this, metric, fields, tags, start, end, columns4RDB, recordSender); + public void sendRecords(List metrics, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender, Map hint) throws Exception { + TSDBDump.dump4RDB(this, metrics, tags, start, end, columns4RDB, recordSender, hint); + } + + @Override + public void sendRecords(String metric, List fields, Map tags, Long start, Long end, List columns4RDB, RecordSender recordSender, Map hint) throws Exception { + TSDBDump.dump4RDB(this, metric, fields, tags, start, end, columns4RDB, recordSender, hint); } @Override diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java index 8bae3a7..c911a06 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBDump.java @@ -9,10 +9,9 @@ import com.alibaba.fastjson.parser.Feature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; + +import static com.alibaba.datax.plugin.reader.tsdbreader.Constant.METRIC_SPECIFY_KEY_PREFIX_LENGTH; /** * Copyright @ 2019 alibaba.com @@ -37,10 +36,10 @@ final class TSDBDump { } static void dump4TSDB(TSDBConnection conn, String metric, Map tags, - Long start, Long end, RecordSender sender) throws Exception { + Long start, Long end, RecordSender sender, Map hint) throws Exception { LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); - String res = queryRange4SingleField(conn, metric, tags, start, end); + String res = queryRange4SingleField(conn, metric, tags, start, end, hint); List dps = getDps4TSDB(metric, res); if (dps == null || dps.isEmpty()) { return; @@ -49,10 +48,10 @@ final class TSDBDump { } static void dump4TSDB(TSDBConnection conn, String metric, List fields, Map tags, - Long start, Long end, RecordSender sender) throws Exception { + Long start, Long end, RecordSender sender, Map hint) throws Exception { LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); - String res = queryRange4MultiFields(conn, metric, fields, tags, start, end); + String res = queryRange4MultiFields(conn, metric, fields, tags, start, end, hint); List dps = getDps4TSDB(metric, fields, res); if (dps == null || dps.isEmpty()) { return; @@ -61,10 +60,10 @@ final class TSDBDump { } static void dump4RDB(TSDBConnection conn, String metric, Map tags, - Long start, Long end, List columns4RDB, RecordSender sender) throws Exception { + Long start, Long end, List columns4RDB, RecordSender sender, Map hint) throws Exception { LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); - String res = queryRange4SingleField(conn, metric, tags, start, end); + String res = queryRange4SingleField(conn, metric, tags, start, end, hint); List dps = getDps4RDB(metric, res); if (dps == null || dps.isEmpty()) { return; @@ -92,12 +91,71 @@ final class TSDBDump { } } + public static void dump4RDB(TSDBConnection conn, List metrics, Map tags, Long start, Long end, List columns4RDB, RecordSender sender, Map hint) throws Exception { + LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metrics, start, end); + + List dps = new LinkedList<>(); + for (String metric : metrics) { + String res = queryRange4SingleField(conn, metric, tags, start, end, hint); + final List dpList = getDps4RDB(metric, res); + if (dpList == null || dpList.isEmpty()) { + continue; + } + dps.addAll(dpList); + } + if (dps.isEmpty()) { + return; + } + Map> dpsCombinedByTs = new LinkedHashMap<>(); + for (DataPoint4TSDB dp : dps) { + final long ts = dp.getTimestamp(); + final Map dpsWithSameTs = dpsCombinedByTs.computeIfAbsent(ts, k -> new LinkedHashMap<>()); + dpsWithSameTs.put(dp.getMetric(), dp); + } + + for (Map.Entry> entry : dpsCombinedByTs.entrySet()) { + final Long ts = entry.getKey(); + final Map metricAndDps = entry.getValue(); + final Record record = sender.createRecord(); + DataPoint4TSDB tmpDp = null; + + for (final String column : columns4RDB) { + if (column.startsWith(Constant.METRIC_SPECIFY_KEY)) { + final String m = column.substring(METRIC_SPECIFY_KEY_PREFIX_LENGTH); + tmpDp = metricAndDps.get(m); + if (tmpDp == null) { + continue; + } + record.addColumn(getColumn(tmpDp.getValue())); + } else if (Constant.TS_SPECIFY_KEY.equals(column)) { + record.addColumn(new LongColumn(ts)); + } else if (Constant.VALUE_SPECIFY_KEY.equals(column)) { + // combine 模式下，不应该定义 __value__ 字段，因为 __metric__.xxx 字段会输出对应的 value 值 + throw new RuntimeException("The " + Constant.VALUE_SPECIFY_KEY + + " column should not be specified in combine mode!"); + } else { + // combine 模式下，应该确保 __metric__.xxx 字段的定义，放在 column 数组的最前面，以保证获取到 metric + if (tmpDp == null) { + throw new RuntimeException("These " + Constant.METRIC_SPECIFY_KEY_PREFIX + + " column should be placed first in the column array in combine mode!"); + } + final Object tagv = tmpDp.getTags().get(column); + if (tagv == null) { + continue; + } + record.addColumn(getColumn(tagv)); + } + } + sender.sendToWriter(record); + } + } + static void dump4RDB(TSDBConnection conn, String metric, List fields, Map tags, Long start, Long end, - List columns4RDB, RecordSender sender) throws Exception { + List columns4RDB, RecordSender sender, Map hint) throws Exception { LOG.info("conn address: {}, metric: {}, start: {}, end: {}", conn.address(), metric, start, end); - String res = queryRange4MultiFields(conn, metric, fields, tags, start, end); + String res = queryRange4MultiFields(conn, metric, fields, tags, start, end, hint); List dps = getDps4RDB(metric, fields, res); if (dps == null || dps.isEmpty()) { return; @@ -131,14 +189,16 @@ final class TSDBDump { valueColumn = new LongColumn((Long) value); } else if (value instanceof String) { valueColumn = new StringColumn((String) value); + } else if (value instanceof Integer) { + valueColumn = new LongColumn(((Integer)value).longValue()); } else { - throw new Exception(String.format("value 不支持类型: [%s]", value.getClass().getSimpleName())); + throw new Exception(String.format("value not supported type: [%s]", value.getClass().getSimpleName())); } return valueColumn; } private static String queryRange4SingleField(TSDBConnection conn, String metric, Map tags, - Long start, Long end) throws Exception { + Long start, Long end, Map hint) throws Exception { String tagKV = getFilterByTags(tags); String body = "{\n" + " \"start\": " + start + ",\n" + @@ -148,14 +208,15 @@ final class TSDBDump { " \"aggregator\": \"none\",\n" + " \"metric\": \"" + metric + "\"\n" + (tagKV == null ? "" : tagKV) + + (hint == null ? "" : (", \"hint\": " + JSON.toJSONString(hint))) + " }\n" + " ]\n" + "}"; - return HttpUtils.post(conn.address() + QUERY, body); + return HttpUtils.post(conn.address() + QUERY, conn.username(), conn.password(), body); } private static String queryRange4MultiFields(TSDBConnection conn, String metric, List fields, - Map tags, Long start, Long end) throws Exception { + Map tags, Long start, Long end, Map hint) throws Exception { // fields StringBuilder fieldBuilder = new StringBuilder(); fieldBuilder.append("\"fields\":["); @@ -177,10 +238,11 @@ final class TSDBDump { " \"metric\": \"" + metric + "\",\n" + fieldBuilder.toString() + (tagKV == null ? "" : tagKV) + + (hint == null ? "" : (", \"hint\": " + JSON.toJSONString(hint))) + " }\n" + " ]\n" + "}"; - return HttpUtils.post(conn.address() + QUERY_MULTI_FIELD, body); + return HttpUtils.post(conn.address() + QUERY_MULTI_FIELD, conn.username(), conn.password(), body); } private static String getFilterByTags(Map tags) { diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java index 3e0be85..5cba4e5 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtils.java @@ -1,11 +1,13 @@ package com.alibaba.datax.plugin.reader.tsdbreader.util; import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; import org.apache.http.client.fluent.Content; import org.apache.http.client.fluent.Request; import org.apache.http.entity.ContentType; import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -22,13 +24,18 @@ public final class HttpUtils { public final static int CONNECT_TIMEOUT_DEFAULT_IN_MILL = (int) TimeUnit.SECONDS.toMillis(60); public final static int SOCKET_TIMEOUT_DEFAULT_IN_MILL = (int) TimeUnit.SECONDS.toMillis(60); + private static final String CREDENTIALS_FORMAT = "%s:%s"; + private static final String BASIC_AUTHENTICATION_FORMAT = "Basic %s"; + private HttpUtils() { } - public static String get(String url) throws Exception { - Content content = Request.Get(url) + public static String get(String url, String username, String password) throws Exception { + final Request request = Request.Get(url) .connectTimeout(CONNECT_TIMEOUT_DEFAULT_IN_MILL) - .socketTimeout(SOCKET_TIMEOUT_DEFAULT_IN_MILL) + .socketTimeout(SOCKET_TIMEOUT_DEFAULT_IN_MILL); + addAuth(request, username, password); + Content content = request .execute() .returnContent(); if (content == null) { @@ -37,24 +44,21 @@ public final class HttpUtils { return content.asString(StandardCharsets.UTF_8); } - public static String post(String url, Map params) throws Exception { - return post(url, JSON.toJSONString(params), CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); + public static String post(String url, String username, String password, Map params) throws Exception { + return post(url, username, password, JSON.toJSONString(params), CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); } - public static String post(String url, String params) throws Exception { - return post(url, params, CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); + public static String post(String url, String username, String password, String params) throws Exception { + return post(url, username, password, params, CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); } - public static String post(String url, Map params, + public static String post(String url, String username, String password, String params, int connectTimeoutInMill, int socketTimeoutInMill) throws Exception { - return post(url, JSON.toJSONString(params), connectTimeoutInMill, socketTimeoutInMill); - } - - public static String post(String url, String params, - int connectTimeoutInMill, int socketTimeoutInMill) throws Exception { - Content content = Request.Post(url) + Request request = Request.Post(url) .connectTimeout(connectTimeoutInMill) - .socketTimeout(socketTimeoutInMill) + .socketTimeout(socketTimeoutInMill); + addAuth(request, username, password); + Content content = request .addHeader("Content-Type", "application/json") .bodyString(params, ContentType.APPLICATION_JSON) .execute() @@ -64,4 +68,20 @@ public final class HttpUtils { } return content.asString(StandardCharsets.UTF_8); } + + private static void addAuth(Request request, String username, String password) { + String authorization = generateHttpAuthorization(username, password); + if (authorization != null) { + request.setHeader("Authorization", authorization); + } + } + + private static String generateHttpAuthorization(String username, String password) { + if (StringUtils.isBlank(username) || StringUtils.isBlank(password)) { + return null; + } + String credentials = String.format(CREDENTIALS_FORMAT, username, password); + credentials = Base64.getEncoder().encodeToString(credentials.getBytes()); + return String.format(BASIC_AUTHENTICATION_FORMAT, credentials); + } } diff --git a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java index bb7b4b8..d91c355 100644 --- a/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java +++ b/tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TSDBUtils.java @@ -1,11 +1,5 @@ package com.alibaba.datax.plugin.reader.tsdbreader.util; -import com.alibaba.datax.plugin.reader.tsdbreader.conn.DataPoint4TSDB; -import com.alibaba.fastjson.JSON; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; /** * Copyright @ 2019 alibaba.com @@ -17,52 +11,28 @@ import java.util.List; */ public final class TSDBUtils { - private static final Logger LOGGER = LoggerFactory.getLogger(TSDBUtils.class); - private TSDBUtils() { } - public static String version(String address) { + public static String version(String address, String username, String password) { String url = String.format("%s/api/version", address); String rsp; try { - rsp = HttpUtils.get(url); + rsp = HttpUtils.get(url, username, password); } catch (Exception e) { throw new RuntimeException(e); } return rsp; } - public static String config(String address) { + public static String config(String address, String username, String password) { String url = String.format("%s/api/config", address); String rsp; try { - rsp = HttpUtils.get(url); + rsp = HttpUtils.get(url, username, password); } catch (Exception e) { throw new RuntimeException(e); } return rsp; } - - public static boolean put(String address, List dps) { - return put(address, JSON.toJSON(dps)); - } - - public static boolean put(String address, DataPoint4TSDB dp) { - return put(address, JSON.toJSON(dp)); - } - - private static boolean put(String address, Object o) { - String url = String.format("%s/api/put", address); - String rsp; - try { - rsp = HttpUtils.post(url, o.toString()); - // If successful, the returned content should be null. - assert rsp == null; - } catch (Exception e) { - LOGGER.error("Address: {}, DataPoints: {}", url, o); - throw new RuntimeException(e); - } - return true; - } } diff --git a/tsdbwriter/pom.xml b/tsdbwriter/pom.xml index 1fb7c1e..fd4cc6f 100644 --- a/tsdbwriter/pom.xml +++ b/tsdbwriter/pom.xml @@ -91,6 +91,13 @@ ${fastjson.version} + + + com.aliyun + hitsdb-client + 0.4.0-SNAPSHOT + + junit diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/Connection4TSDB.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/Connection4TSDB.java index 8119348..ecb3005 100644 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/Connection4TSDB.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/Connection4TSDB.java @@ -21,6 +21,28 @@ public interface Connection4TSDB { */ String address(); + /** + * Get the setted database name. + * + * @return database + */ + String database(); + + + /** + * Get the username of Database. + * + * @return username + */ + String username(); + + /** + * Get the password of Database. + * + * @return password + */ + String password(); + /** * Get the version of Database. * @@ -69,17 +91,25 @@ public interface Connection4TSDB { boolean put(List dps); /** - * Put data points. + * Put data points with single field. * * @param dps data points * @return whether the data point is written successfully */ boolean put(String dps); + /** + * Put data points with multi fields. + * + * @param dps data points + * @return whether the data point is written successfully + */ + boolean mput(String dps); + /** * Whether current version is supported. * * @return true: supported; false: not yet! */ boolean isSupported(); -} +} \ No newline at end of file diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/TSDBConnection.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/TSDBConnection.java index e4ebad7..074f029 100644 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/TSDBConnection.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/conn/TSDBConnection.java @@ -18,12 +18,18 @@ import java.util.List; public class TSDBConnection implements Connection4TSDB { private String address; + private String username; + private String password; + private String database; - public TSDBConnection(String address) { + public TSDBConnection(String address, String database, String username, String password) { if (StringUtils.isBlank(address)) { throw new RuntimeException("TSDBConnection init failed because address is blank!"); } this.address = address; + this.database = database; + this.username = username; + this.password = password; } @Override @@ -31,14 +37,29 @@ public class TSDBConnection implements Connection4TSDB { return address; } + @Override + public String username() { + return username; + } + + @Override + public String database() { + return database; + } + + @Override + public String password() { + return password; + } + @Override public String version() { - return TSDBUtils.version(address); + return TSDBUtils.version(address, username, password); } @Override public String config() { - return TSDBUtils.config(address); + return TSDBUtils.config(address, username, password); } @Override @@ -53,17 +74,22 @@ public class TSDBConnection implements Connection4TSDB { @Override public boolean put(DataPoint4TSDB dp) { - return TSDBUtils.put(address, dp); + return TSDBUtils.put(address, database, username, password, dp); } @Override public boolean put(List dps) { - return TSDBUtils.put(address, dps); + return TSDBUtils.put(address, database, username, password, dps); } @Override public boolean put(String dps) { - return TSDBUtils.put(address, dps); + return TSDBUtils.put(address, database, username, password, dps); + } + + @Override + public boolean mput(String dps) { + return TSDBUtils.mput(address, database, username, password, dps); } @Override diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/Key.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/Key.java index 2cc3f67..6cb239e 100755 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/Key.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/Key.java @@ -10,8 +10,22 @@ package com.alibaba.datax.plugin.writer.tsdbwriter; */ public class Key { + static final String SOURCE_DB_TYPE = "sourceDbType"; + static final String MULTI_FIELD = "multiField"; + + // common static final String ENDPOINT = "endpoint"; + static final String USERNAME = "username"; + static final String PASSWORD = "password"; + static final String IGNORE_WRITE_ERROR = "ignoreWriteError"; + static final String DATABASE = "database"; + + // for tsdb static final String BATCH_SIZE = "batchSize"; static final String MAX_RETRY_TIME = "maxRetryTime"; - static final String IGNORE_WRITE_ERROR = "ignoreWriteError"; + + // for rdb + static final String COLUMN = "column"; + static final String COLUMN_TYPE = "columnType"; + static final String TABLE = "table"; } diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/SourceDBType.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/SourceDBType.java new file mode 100644 index 0000000..792806a --- /dev/null +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/SourceDBType.java @@ -0,0 +1,5 @@ +package com.alibaba.datax.plugin.writer.tsdbwriter; + +public enum SourceDBType { + TSDB, RDB +} diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBConverter.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBConverter.java new file mode 100644 index 0000000..86e35c5 --- /dev/null +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBConverter.java @@ -0,0 +1,96 @@ +package com.alibaba.datax.plugin.writer.tsdbwriter; + +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.fastjson.JSON; +import com.aliyun.hitsdb.client.value.request.MultiFieldPoint; +import com.aliyun.hitsdb.client.value.request.Point; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class TSDBConverter { + + private static final Logger LOG = LoggerFactory.getLogger(TSDBConverter.class); + + private List columnName; + private List columnType; + + TSDBConverter(List columnName, List columnType) { + this.columnName = columnName; + this.columnType = columnType; + LOG.info("columnName: {}, columnType: {}", JSON.toJSONString(columnName), JSON.toJSONString(columnType)); + } + + List transRecord2Point(List records) { + List dps = new ArrayList(); + for (Record record : records) { + List metricBuilders = new ArrayList(); + Map tags = new HashMap(); + Long time = 0L; + + for (int i = 0; i < columnType.size(); i++) { + String type = columnType.get(i); + String name = columnName.get(i); + Column column = record.getColumn(i); + if (TSDBModel.TSDB_TAG.equals(type)) { + tags.put(name, column.asString()); + } else if (TSDBModel.TSDB_FIELD_DOUBLE.equals(type)) { + metricBuilders.add(new Point.MetricBuilder(name).value(column.asDouble())); + } else if (TSDBModel.TSDB_FIELD_STRING.equals(type)) { + metricBuilders.add(new Point.MetricBuilder(name).value(column.asString())); + } else if (TSDBModel.TSDB_FIELD_BOOL.equals(type)) { + metricBuilders.add(new Point.MetricBuilder(name).value(column.asBoolean())); + } else if (TSDBModel.TSDB_TIMESTAMP.equals(type)) { + time = column.asLong(); + } else if (TSDBModel.TSDB_METRIC_NUM.equals(type)) { + // compatible with previous usage of TSDB_METRIC_NUM + metricBuilders.add(new Point.MetricBuilder(name).value(column.asDouble())); + } else if (TSDBModel.TSDB_METRIC_STRING.equals(type)) { + // compatible with previous usage of TSDB_METRIC_STRING + metricBuilders.add(new Point.MetricBuilder(name).value(column.asString())); + } + } + for (Point.MetricBuilder metricBuilder : metricBuilders) { + dps.add(metricBuilder.tag(tags).timestamp(time).build(false)); + } + } + return dps; + } + + List transRecord2MultiFieldPoint(List records, String tableName) { + List dps = new ArrayList(); + for (Record record : records) { + MultiFieldPoint.MetricBuilder builder = MultiFieldPoint.metric(tableName); + for (int i = 0; i < columnType.size(); i++) { + String type = columnType.get(i); + String name = columnName.get(i); + Column column = record.getColumn(i); + if (TSDBModel.TSDB_TAG.equals(type)) { + builder.tag(name, column.asString()); + } else if (TSDBModel.TSDB_FIELD_DOUBLE.equals(type)) { + builder.field(name, column.asDouble()); + } else if (TSDBModel.TSDB_FIELD_STRING.equals(type)) { + builder.field(name, column.asString()); + } else if (TSDBModel.TSDB_FIELD_BOOL.equals(type)) { + builder.field(name, column.asBoolean()); + } else if (TSDBModel.TSDB_TIMESTAMP.equals(type)) { + builder.timestamp(column.asLong()); + } else if (TSDBModel.TSDB_METRIC_NUM.equals(type)) { + // compatible with previous usage of TSDB_METRIC_NUM + builder.field(name, column.asDouble()); + } else if (TSDBModel.TSDB_METRIC_STRING.equals(type)) { + // compatible with previous usage of TSDB_METRIC_STRING + builder.field(name, column.asString()); + } + } + MultiFieldPoint point = builder.build(false); + dps.add(point); + } + return dps; + } +} diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBModel.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBModel.java new file mode 100644 index 0000000..ead0e2c --- /dev/null +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBModel.java @@ -0,0 +1,11 @@ +package com.alibaba.datax.plugin.writer.tsdbwriter; + +class TSDBModel { + static final String TSDB_METRIC_NUM = "metric_num"; + static final String TSDB_METRIC_STRING = "metric_string"; + static final String TSDB_TAG = "tag"; + static final String TSDB_TIMESTAMP = "timestamp"; + static final String TSDB_FIELD_DOUBLE = "field_double"; + static final String TSDB_FIELD_STRING = "field_string"; + static final String TSDB_FIELD_BOOL = "field_bool"; +} \ No newline at end of file diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriter.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriter.java index e410b2b..85a32a0 100755 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriter.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriter.java @@ -7,12 +7,20 @@ import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.common.util.RetryUtil; import com.alibaba.datax.plugin.writer.conn.TSDBConnection; +import com.aliyun.hitsdb.client.TSDB; +import com.aliyun.hitsdb.client.TSDBClientFactory; +import com.aliyun.hitsdb.client.TSDBConfig; +import com.aliyun.hitsdb.client.value.request.MultiFieldPoint; +import com.aliyun.hitsdb.client.value.request.Point; +import com.aliyun.hitsdb.client.value.response.batch.IgnoreErrorsResult; +import com.aliyun.hitsdb.client.value.response.batch.MultiFieldIgnoreErrorsResult; +import com.aliyun.hitsdb.client.value.response.batch.SummaryResult; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; +import java.io.IOException; +import java.util.*; import java.util.concurrent.Callable; /** @@ -26,6 +34,9 @@ import java.util.concurrent.Callable; @SuppressWarnings("unused") public class TSDBWriter extends Writer { + private static SourceDBType DB_TYPE; + private static TSDB tsdb = null; + public static class Job extends Writer.Job { private static final Logger LOG = LoggerFactory.getLogger(Job.class); @@ -34,33 +45,99 @@ public class TSDBWriter extends Writer { @Override public void init() { - this.originalConfig = super.getPluginJobConf(); + originalConfig = super.getPluginJobConf(); - String address = this.originalConfig.getString(Key.ENDPOINT); - if (StringUtils.isBlank(address)) { + // check source db type + String sourceDbType = originalConfig.getString(Key.SOURCE_DB_TYPE); + if (StringUtils.isBlank(sourceDbType)) { + sourceDbType = SourceDBType.TSDB.name(); + originalConfig.set(Key.SOURCE_DB_TYPE, sourceDbType); + LOG.info("The parameter [" + Key.SOURCE_DB_TYPE + "] will be default value: " + SourceDBType.TSDB); + } + try { + DB_TYPE = SourceDBType.valueOf(sourceDbType); + } catch (Exception e) { throw DataXException.asDataXException(TSDBWriterErrorCode.REQUIRED_VALUE, - "The parameter [" + Key.ENDPOINT + "] is not set."); + "The parameter [" + Key.SOURCE_DB_TYPE + + "] is invalid, which should be one of [" + Arrays.toString(SourceDBType.values()) + "]."); } - Integer batchSize = this.originalConfig.getInt(Key.BATCH_SIZE); - if (batchSize == null || batchSize < 1) { - originalConfig.set(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE); - LOG.info("The parameter [" + Key.BATCH_SIZE + - "] will be default value: " + Constant.DEFAULT_BATCH_SIZE); - } + // for tsdb + if (DB_TYPE == SourceDBType.TSDB) { + String address = originalConfig.getString(Key.ENDPOINT); + if (StringUtils.isBlank(address)) { + throw DataXException.asDataXException(TSDBWriterErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.ENDPOINT + "] is not set."); + } - Integer retrySize = this.originalConfig.getInt(Key.MAX_RETRY_TIME); - if (retrySize == null || retrySize < 0) { - originalConfig.set(Key.MAX_RETRY_TIME, Constant.DEFAULT_TRY_SIZE); - LOG.info("The parameter [" + Key.MAX_RETRY_TIME + - "] will be default value: " + Constant.DEFAULT_TRY_SIZE); - } + String username = originalConfig.getString(Key.USERNAME, null); + if (StringUtils.isBlank(username)) { + LOG.warn("The parameter [" + Key.USERNAME + "] is blank."); + } + String password = originalConfig.getString(Key.PASSWORD, null); + if (StringUtils.isBlank(password)) { + LOG.warn("The parameter [" + Key.PASSWORD + "] is blank."); + } + + Integer batchSize = originalConfig.getInt(Key.BATCH_SIZE); + if (batchSize == null || batchSize < 1) { + originalConfig.set(Key.BATCH_SIZE, Constant.DEFAULT_BATCH_SIZE); + LOG.info("The parameter [" + Key.BATCH_SIZE + + "] will be default value: " + Constant.DEFAULT_BATCH_SIZE); + } + + Integer retrySize = originalConfig.getInt(Key.MAX_RETRY_TIME); + if (retrySize == null || retrySize < 0) { + originalConfig.set(Key.MAX_RETRY_TIME, Constant.DEFAULT_TRY_SIZE); + LOG.info("The parameter [" + Key.MAX_RETRY_TIME + + "] will be default value: " + Constant.DEFAULT_TRY_SIZE); + } + + Boolean ignoreWriteError = originalConfig.getBool(Key.IGNORE_WRITE_ERROR); + if (ignoreWriteError == null) { + originalConfig.set(Key.IGNORE_WRITE_ERROR, Constant.DEFAULT_IGNORE_WRITE_ERROR); + LOG.info("The parameter [" + Key.IGNORE_WRITE_ERROR + + "] will be default value: " + Constant.DEFAULT_IGNORE_WRITE_ERROR); + } + } else if (DB_TYPE == SourceDBType.RDB) { + // for rdb + originalConfig.getNecessaryValue(Key.ENDPOINT, TSDBWriterErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(Key.COLUMN_TYPE, TSDBWriterErrorCode.REQUIRED_VALUE); + originalConfig.getNecessaryValue(Key.COLUMN, TSDBWriterErrorCode.REQUIRED_VALUE); + String endpoint = originalConfig.getString(Key.ENDPOINT); + String[] split = endpoint.split(":"); + if (split.length != 3) { + throw DataXException.asDataXException(TSDBWriterErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.ENDPOINT + "] is invalid, which should be [http://IP:Port]."); + } + String ip = split[1].substring(2); + int port = Integer.parseInt(split[2]); + + String username = originalConfig.getString(Key.USERNAME, null); + if (StringUtils.isBlank(username)) { + LOG.warn("The parameter [" + Key.USERNAME + "] is blank."); + } + + String password = originalConfig.getString(Key.PASSWORD, null); + if (StringUtils.isBlank(password)) { + LOG.warn("The parameter [" + Key.PASSWORD + "] is blank."); + } + + if (!StringUtils.isBlank(password) && !StringUtils.isBlank(username)) { + tsdb = TSDBClientFactory.connect(TSDBConfig.address(ip, port).basicAuth(username, password).config()); + } else { + tsdb = TSDBClientFactory.connect(TSDBConfig.address(ip, port).config()); + } + + String database = originalConfig.getString(Key.DATABASE, null); + if (StringUtils.isBlank(database)) { + LOG.info("The parameter [" + Key.DATABASE + "] is blank."); + } else { + tsdb.useDatabase(database); + } + + LOG.info("Tsdb config:" + originalConfig.toJSON()); - Boolean ignoreWriteError = this.originalConfig.getBool(Key.IGNORE_WRITE_ERROR); - if (ignoreWriteError == null) { - originalConfig.set(Key.IGNORE_WRITE_ERROR, Constant.DEFAULT_IGNORE_WRITE_ERROR); - LOG.info("The parameter [" + Key.IGNORE_WRITE_ERROR + - "] will be default value: " + Constant.DEFAULT_IGNORE_WRITE_ERROR); } } @@ -72,7 +149,7 @@ public class TSDBWriter extends Writer { public List split(int mandatoryNumber) { ArrayList configurations = new ArrayList(mandatoryNumber); for (int i = 0; i < mandatoryNumber; i++) { - configurations.add(this.originalConfig.clone()); + configurations.add(originalConfig.clone()); } return configurations; } @@ -83,6 +160,14 @@ public class TSDBWriter extends Writer { @Override public void destroy() { + if (DB_TYPE == SourceDBType.RDB) { + if (tsdb != null) { + try { + tsdb.close(); + } catch (IOException ignored) { + } + } + } } } @@ -91,18 +176,87 @@ public class TSDBWriter extends Writer { private static final Logger LOG = LoggerFactory.getLogger(Task.class); private TSDBConnection conn; + private boolean multiField; private int batchSize; private int retrySize; private boolean ignoreWriteError; + private String tableName; + private TSDBConverter tsdbConverter; @Override public void init() { Configuration writerSliceConfig = getPluginJobConf(); - String address = writerSliceConfig.getString(Key.ENDPOINT); - this.conn = new TSDBConnection(address); - this.batchSize = writerSliceConfig.getInt(Key.BATCH_SIZE); - this.retrySize = writerSliceConfig.getInt(Key.MAX_RETRY_TIME); + + // single field | multi fields + this.multiField = writerSliceConfig.getBool(Key.MULTI_FIELD, false); this.ignoreWriteError = writerSliceConfig.getBool(Key.IGNORE_WRITE_ERROR); + + // for tsdb + if (DB_TYPE == SourceDBType.TSDB) { + String address = writerSliceConfig.getString(Key.ENDPOINT); + String database = writerSliceConfig.getString(Key.DATABASE); + String username = writerSliceConfig.getString(Key.USERNAME); + String password = writerSliceConfig.getString(Key.PASSWORD); + this.conn = new TSDBConnection(address, database, username, password); + this.batchSize = writerSliceConfig.getInt(Key.BATCH_SIZE); + this.retrySize = writerSliceConfig.getInt(Key.MAX_RETRY_TIME); + + } else if (DB_TYPE == SourceDBType.RDB) { + // for rdb + int timeSize = 0; + int fieldSize = 0; + int tagSize = 0; + batchSize = writerSliceConfig.getInt(Key.BATCH_SIZE, 100); + List columnName = writerSliceConfig.getList(Key.COLUMN, String.class); + List columnType = writerSliceConfig.getList(Key.COLUMN_TYPE, String.class); + Set typeSet = new HashSet(columnType); + if (columnName.size() != columnType.size()) { + throw DataXException.asDataXException(TSDBWriterErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.COLUMN_TYPE + "] should has same length with [" + Key.COLUMN + "]."); + } + + for (String type : columnType) { + if (TSDBModel.TSDB_TAG.equals(type)) { + tagSize ++; + } else if (TSDBModel.TSDB_FIELD_DOUBLE.equals(type) || TSDBModel.TSDB_FIELD_STRING.equals(type) + || TSDBModel.TSDB_FIELD_BOOL.equals(type)) { + fieldSize++; + } else if (TSDBModel.TSDB_TIMESTAMP.equals(type)) { + timeSize++; + } + } + + if (fieldSize == 0) { + // compatible with previous usage of TSDB_METRIC_NUM and TSDB_METRIC_STRING + if (!typeSet.contains(TSDBModel.TSDB_METRIC_NUM) && !typeSet.contains(TSDBModel.TSDB_METRIC_STRING)) { + throw DataXException.asDataXException(TSDBWriterErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.COLUMN_TYPE + "] is invalid, must set at least one of " + + TSDBModel.TSDB_FIELD_DOUBLE + ", " + TSDBModel.TSDB_FIELD_STRING + " or " + TSDBModel.TSDB_FIELD_BOOL + "."); + } + } + + if (tagSize == 0) { + throw DataXException.asDataXException(TSDBWriterErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.COLUMN_TYPE + "] is invalid, must set " + TSDBModel.TSDB_TAG + ". "); + } + + if (timeSize != 1) { + throw DataXException.asDataXException(TSDBWriterErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.COLUMN_TYPE + "] is invalid, must set one and only one " + + TSDBModel.TSDB_TIMESTAMP + "."); + } + + if (multiField) { + // check source db type + tableName = writerSliceConfig.getString(Key.TABLE); + if (StringUtils.isBlank(tableName)) { + throw DataXException.asDataXException(TSDBWriterErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.TABLE + "] h must set when use multi field input."); + } + } + tsdbConverter = new TSDBConverter(columnName, columnType); + + } } @Override @@ -111,30 +265,52 @@ public class TSDBWriter extends Writer { @Override public void startWrite(RecordReceiver recordReceiver) { - try { - Record lastRecord = null; - Record record; - int count = 0; - StringBuilder dps = new StringBuilder(); - while ((record = recordReceiver.getFromReader()) != null) { - final int recordLength = record.getColumnNumber(); - for (int i = 0; i < recordLength; i++) { - dps.append(record.getColumn(i).asString()); - dps.append(","); - count++; - if (count == batchSize) { - count = 0; - batchPut(record, "[" + dps.substring(0, dps.length() - 1) + "]"); - dps = new StringBuilder(); + // for tsdb + if (DB_TYPE == SourceDBType.TSDB) { + try { + Record lastRecord = null; + Record record; + int count = 0; + StringBuilder dps = new StringBuilder(); + while ((record = recordReceiver.getFromReader()) != null) { + final int recordLength = record.getColumnNumber(); + for (int i = 0; i < recordLength; i++) { + dps.append(record.getColumn(i).asString()); + dps.append(","); + count++; + if (count == batchSize) { + count = 0; + batchPut(record, "[" + dps.substring(0, dps.length() - 1) + "]"); + dps = new StringBuilder(); + } } + lastRecord = record; } - lastRecord = record; + if (StringUtils.isNotBlank(dps.toString())) { + batchPut(lastRecord, "[" + dps.substring(0, dps.length() - 1) + "]"); + } + } catch (Exception e) { + throw DataXException.asDataXException(TSDBWriterErrorCode.RUNTIME_EXCEPTION, e); } - if (StringUtils.isNotBlank(dps.toString())) { - batchPut(lastRecord, "[" + dps.substring(0, dps.length() - 1) + "]"); + } else if (DB_TYPE == SourceDBType.RDB) { + // for rdb + List writerBuffer = new ArrayList(this.batchSize); + Record record; + long total = 0; + while ((record = recordReceiver.getFromReader()) != null) { + writerBuffer.add(record); + if (writerBuffer.size() >= this.batchSize) { + total += doBatchInsert(writerBuffer); + writerBuffer.clear(); + } } - } catch (Exception e) { - throw DataXException.asDataXException(TSDBWriterErrorCode.RUNTIME_EXCEPTION, e); + if (!writerBuffer.isEmpty()) { + total += doBatchInsert(writerBuffer); + writerBuffer.clear(); + } + getTaskPluginCollector().collectMessage("write size", total + ""); + LOG.info("Task finished, write size: {}", total); + } } @@ -143,12 +319,13 @@ public class TSDBWriter extends Writer { RetryUtil.executeWithRetry(new Callable() { @Override public Integer call() { - if (!conn.put(dps)) { - getTaskPluginCollector().collectDirtyRecord(record, "Put data points failed!"); - throw DataXException.asDataXException(TSDBWriterErrorCode.RUNTIME_EXCEPTION, - "Put data points failed!"); + final boolean success = multiField ? conn.mput(dps) : conn.put(dps); + if (success) { + return 0; } - return 0; + getTaskPluginCollector().collectDirtyRecord(record, "Put data points failed!"); + throw DataXException.asDataXException(TSDBWriterErrorCode.RUNTIME_EXCEPTION, + "Put data points failed!"); } }, retrySize, 60000L, true); } catch (Exception e) { @@ -160,6 +337,47 @@ public class TSDBWriter extends Writer { } } + private long doBatchInsert(final List writerBuffer) { + int size; + if (ignoreWriteError) { + if (multiField) { + List points = tsdbConverter.transRecord2MultiFieldPoint(writerBuffer, tableName); + size = points.size(); + MultiFieldIgnoreErrorsResult ignoreErrorsResult = tsdb.multiFieldPutSync(points, MultiFieldIgnoreErrorsResult.class); + if (ignoreErrorsResult == null) { + LOG.error("Unexpected inner error for insert"); + } else if (ignoreErrorsResult.getFailed() > 0) { + LOG.error("write TSDB failed num:" + ignoreErrorsResult.getFailed()); + } + } else { + List points = tsdbConverter.transRecord2Point(writerBuffer); + size = points.size(); + IgnoreErrorsResult ignoreErrorsResult = tsdb.putSync(points, IgnoreErrorsResult.class); + if (ignoreErrorsResult == null) { + LOG.error("Unexpected inner error for insert"); + } else if (ignoreErrorsResult.getFailed() > 0) { + LOG.error("write TSDB failed num:" + ignoreErrorsResult.getFailed()); + } + } + } else { + SummaryResult summaryResult; + if (multiField) { + List points = tsdbConverter.transRecord2MultiFieldPoint(writerBuffer, tableName); + size = points.size(); + summaryResult = tsdb.multiFieldPutSync(points, SummaryResult.class); + } else { + List points = tsdbConverter.transRecord2Point(writerBuffer); + size = points.size(); + summaryResult = tsdb.putSync(points, SummaryResult.class); + } + if (summaryResult.getFailed() > 0) { + LOG.error("write TSDB failed num:" + summaryResult.getFailed()); + throw DataXException.asDataXException(TSDBWriterErrorCode.RUNTIME_EXCEPTION, "Write TSDB failed", new Exception()); + } + } + return size; + } + @Override public void post() { } diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriterErrorCode.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriterErrorCode.java index f907fb6..ab4c389 100755 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriterErrorCode.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/TSDBWriterErrorCode.java @@ -13,6 +13,7 @@ import com.alibaba.datax.common.spi.ErrorCode; public enum TSDBWriterErrorCode implements ErrorCode { REQUIRED_VALUE("TSDBWriter-00", "Missing the necessary value"), + ILLEGAL_VALUE("TSDBWriter-01", "Illegal value"), RUNTIME_EXCEPTION("TSDBWriter-01", "Runtime exception"), RETRY_WRITER_EXCEPTION("TSDBWriter-02", "After repeated attempts, the write still fails"); diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/HttpUtils.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/HttpUtils.java index b81512f..29b14da 100644 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/HttpUtils.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/HttpUtils.java @@ -1,11 +1,14 @@ package com.alibaba.datax.plugin.writer.util; import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; import org.apache.http.client.fluent.Content; import org.apache.http.client.fluent.Request; import org.apache.http.entity.ContentType; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -19,43 +22,44 @@ import java.util.concurrent.TimeUnit; */ public final class HttpUtils { - public final static Charset UTF_8 = Charset.forName("UTF-8"); public final static int CONNECT_TIMEOUT_DEFAULT_IN_MILL = (int) TimeUnit.SECONDS.toMillis(60); public final static int SOCKET_TIMEOUT_DEFAULT_IN_MILL = (int) TimeUnit.SECONDS.toMillis(60); + private static final String CREDENTIALS_FORMAT = "%s:%s"; + private static final String BASIC_AUTHENTICATION_FORMAT = "Basic %s"; + private HttpUtils() { } - public static String get(String url) throws Exception { - Content content = Request.Get(url) + public static String get(String url, String username, String password) throws Exception { + final Request request = Request.Get(url) .connectTimeout(CONNECT_TIMEOUT_DEFAULT_IN_MILL) - .socketTimeout(SOCKET_TIMEOUT_DEFAULT_IN_MILL) + .socketTimeout(SOCKET_TIMEOUT_DEFAULT_IN_MILL); + addAuth(request, username, password); + Content content = request .execute() .returnContent(); if (content == null) { return null; } - return content.asString(UTF_8); + return content.asString(StandardCharsets.UTF_8); } - public static String post(String url, Map params) throws Exception { - return post(url, JSON.toJSONString(params), CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); + public static String post(String url, String username, String password, Map params) throws Exception { + return post(url, username, password, JSON.toJSONString(params), CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); } - public static String post(String url, String params) throws Exception { - return post(url, params, CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); + public static String post(String url, String username, String password, String params) throws Exception { + return post(url, username, password, params, CONNECT_TIMEOUT_DEFAULT_IN_MILL, SOCKET_TIMEOUT_DEFAULT_IN_MILL); } - public static String post(String url, Map params, + public static String post(String url, String username, String password, String params, int connectTimeoutInMill, int socketTimeoutInMill) throws Exception { - return post(url, JSON.toJSONString(params), connectTimeoutInMill, socketTimeoutInMill); - } - - public static String post(String url, String params, - int connectTimeoutInMill, int socketTimeoutInMill) throws Exception { - Content content = Request.Post(url) + Request request = Request.Post(url) .connectTimeout(connectTimeoutInMill) - .socketTimeout(socketTimeoutInMill) + .socketTimeout(socketTimeoutInMill); + addAuth(request, username, password); + Content content = request .addHeader("Content-Type", "application/json") .bodyString(params, ContentType.APPLICATION_JSON) .execute() @@ -63,6 +67,22 @@ public final class HttpUtils { if (content == null) { return null; } - return content.asString(UTF_8); + return content.asString(StandardCharsets.UTF_8); + } + + private static void addAuth(Request request, String username, String password) { + String authorization = generateHttpAuthorization(username, password); + if (authorization != null) { + request.setHeader("Authorization", authorization); + } + } + + private static String generateHttpAuthorization(String username, String password) { + if (StringUtils.isBlank(username) || StringUtils.isBlank(password)) { + return null; + } + String credentials = String.format(CREDENTIALS_FORMAT, username, password); + credentials = Base64.getEncoder().encodeToString(credentials.getBytes()); + return String.format(BASIC_AUTHENTICATION_FORMAT, credentials); } } diff --git a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/TSDBUtils.java b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/TSDBUtils.java index ed01d87..d57c593 100644 --- a/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/TSDBUtils.java +++ b/tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/util/TSDBUtils.java @@ -2,6 +2,7 @@ package com.alibaba.datax.plugin.writer.util; import com.alibaba.datax.plugin.writer.conn.DataPoint4TSDB; import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,45 +23,56 @@ public final class TSDBUtils { private TSDBUtils() { } - public static String version(String address) { + public static String version(String address, String username, String password) { String url = String.format("%s/api/version", address); String rsp; try { - rsp = HttpUtils.get(url); + rsp = HttpUtils.get(url, username, password); } catch (Exception e) { throw new RuntimeException(e); } return rsp; } - public static String config(String address) { + public static String config(String address, String username, String password) { String url = String.format("%s/api/config", address); String rsp; try { - rsp = HttpUtils.get(url); + rsp = HttpUtils.get(url, username, password); } catch (Exception e) { throw new RuntimeException(e); } return rsp; } - public static boolean put(String address, List dps) { - return put(address, JSON.toJSON(dps)); + public static boolean put(String address, String database, String username, String password, List dps) { + return put(address, database, username, password, JSON.toJSON(dps)); } - public static boolean put(String address, DataPoint4TSDB dp) { - return put(address, JSON.toJSON(dp)); + public static boolean put(String address, String database, String username, String password, DataPoint4TSDB dp) { + return put(address, database, username, password, JSON.toJSON(dp)); } - private static boolean put(String address, Object o) { - return put(address, o.toString()); + private static boolean put(String address, String database, String username, String password, Object o) { + return put(address, database, username, password, o.toString()); } - public static boolean put(String address, String s) { - String url = String.format("%s/api/put", address); + public static boolean put(String address, String database, String username, String password, String s) { + return put(address, database, username, password, s, false); + } + + public static boolean mput(String address, String database, String username, String password, String s) { + return put(address, database, username, password, s, true); + } + + public static boolean put(String address, String database, String username, String password, String s, boolean multiField) { + String url = address + (multiField ? "/api/mput" : "/api/put"); + if (!StringUtils.isBlank(database)) { + url = url.concat("?db=" + database); + } String rsp; try { - rsp = HttpUtils.post(url, s); + rsp = HttpUtils.post(url, username, password, s); // If successful, the returned content should be null. assert rsp == null; } catch (Exception e) { From f2f4a1cf5e3ac5fe418ac67b739c9beffeeb7d8b Mon Sep 17 00:00:00 2001 From: dingbo Date: Wed, 15 Dec 2021 15:31:03 +0800 Subject: [PATCH 065/114] Use UTF-8 as default charset --- .../plugin/writer/tdenginewriter/DefaultDataHandler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 91c2b7e..a8704f2 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -3,6 +3,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBPreparedStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,8 +77,10 @@ public class DefaultDataHandler implements DataHandler { return null; } String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", host, port, dbname, user, password); + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); LOG.info("TDengine connection established, host:{} port:{} dbname:{} user:{}", host, port, dbname, user); - return DriverManager.getConnection(jdbcUrl); + return DriverManager.getConnection(jdbcUrl, connProps); } /** From 6d27133ff4a1be5735d8a8ceb19b5d5cabf3f333 Mon Sep 17 00:00:00 2001 From: dingbo Date: Wed, 15 Dec 2021 15:31:03 +0800 Subject: [PATCH 066/114] Use UTF-8 as default charset --- .../plugin/writer/tdenginewriter/DefaultDataHandler.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 91c2b7e..a8704f2 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -3,6 +3,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBPreparedStatement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,8 +77,10 @@ public class DefaultDataHandler implements DataHandler { return null; } String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", host, port, dbname, user, password); + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); LOG.info("TDengine connection established, host:{} port:{} dbname:{} user:{}", host, port, dbname, user); - return DriverManager.getConnection(jdbcUrl); + return DriverManager.getConnection(jdbcUrl, connProps); } /** From 12483fe75a9d064ffca31d8493069687ac89aa5c Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 16 Dec 2021 11:36:44 +0800 Subject: [PATCH 067/114] [TD-10786]: tdenginereader implementation --- package.xml | 7 + pom.xml | 1 + tdenginereader/doc/tdenginereader.md | 162 ++++++++++ tdenginereader/pom.xml | 101 ++++++ tdenginereader/src/main/assembly/package.xml | 34 ++ .../com/alibaba/datax/plugin/reader/Key.java | 19 ++ .../datax/plugin/reader/TDengineReader.java | 291 ++++++++++++++++++ .../reader/TDengineReaderErrorCode.java | 33 ++ tdenginereader/src/main/resources/plugin.json | 9 + .../main/resources/plugin_job_template.json | 14 + .../plugin/reader/TDengineReaderTest.java | 47 +++ 11 files changed, 718 insertions(+) create mode 100644 tdenginereader/doc/tdenginereader.md create mode 100644 tdenginereader/pom.xml create mode 100755 tdenginereader/src/main/assembly/package.xml create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java create mode 100755 tdenginereader/src/main/resources/plugin.json create mode 100644 tdenginereader/src/main/resources/plugin_job_template.json create mode 100644 tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java diff --git a/package.xml b/package.xml index d95feb4..3f024f9 100755 --- a/package.xml +++ b/package.xml @@ -180,6 +180,13 @@ datax + + tdenginereader/target/datax/ + + **/*.* + + datax + diff --git a/pom.xml b/pom.xml index 1a9da81..7224113 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,7 @@ hbase20xsqlreader hbase20xsqlwriter kuduwriter + tdenginereader diff --git a/tdenginereader/doc/tdenginereader.md b/tdenginereader/doc/tdenginereader.md new file mode 100644 index 0000000..3c683a6 --- /dev/null +++ b/tdenginereader/doc/tdenginereader.md @@ -0,0 +1,162 @@ +# DataX TDengineReader + +## 1 快速介绍 + +TDengineReader 插件实现了 TDengine 读取数据的功能。 + +## 2 实现原理 + +TDengineReader 通过TDengine的JDBC driver查询获取数据。 + +## 3 功能说明 + +### 3.1 配置样例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "user": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "meters" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "ts", + "current", + "voltage", + "phase" + ], + "beginDateTime": "2017-07-14 10:40:00", + "endDateTime": "2017-08-14 10:40:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +### 3.2 参数说明 + +* **username** + * 描述：TDengine实例的用户名
+ * 必选：是
+ * 默认值：无
+* **password** + * 描述：TDengine实例的密码
+ * 必选：是
+ * 默认值：无
+* **table** + * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构， + TDengineReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
+ * 必选：是
+ * 默认值：无
+* **jdbcUrl** + * 描述：TDengine数据库的JDBC连接信息。注意，jdbcUrl必须包含在connection配置单元中。JdbcUrl具体请参看TDengine官方文档。 + * 必选：是
+ * 默认值：无
+* **beginDateTime** + * 描述：数据的开始时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss
+ * 必选：是
+ * 默认值：无
+* **endDateTime** + * 描述：数据的结束时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss
+ * 必选：是
+ * 默认值：无
+* **splitInterval** + * 描述：按照splitInterval来划分task, 每splitInterval创建一个task。例如，20d代表按照每20天的数据划分为1个task。 + 可以配置的时间单位为：d（天）, h（小时）, m（分钟）, s（秒）
+ * 必选：是
+ * 默认值：无
+ +### 3.3 类型转换 +| TDengine 数据类型 | DataX 内部类型 | +| --------------- | ------------- | +| TINYINT | Long | +| SMALLINT | Long | +| INTEGER | Long | +| BIGINT | Long | +| FLOAT | Double | +| DOUBLE | Double | +| BOOLEAN | Bool | +| TIMESTAMP | Date | +| BINARY | Bytes | +| NCHAR | String | + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡流出流量(MB/s)|DataX机器运行负载|DB网卡进入流量(MB/s)|DB运行负载|DB TPS| +|--------| --------|--------|--------|--------|--------|--------|--------| +|1| | | | | | | | +|4| | | | | | | | +|8| | | | | | | | +|16| | | | | | | | +|32| | | | | | | | + +说明： + +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 + +#### 4.2.4 性能测试小结 + +1. +2. + +## 5 约束限制 + +## FAQ \ No newline at end of file diff --git a/tdenginereader/pom.xml b/tdenginereader/pom.xml new file mode 100644 index 0000000..8c0f664 --- /dev/null +++ b/tdenginereader/pom.xml @@ -0,0 +1,101 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + tdenginereader + + + 8 + 8 + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + + + + junit + junit + ${junit-version} + test + + + com.alibaba.datax + plugin-rdbms-util + 0.0.1-SNAPSHOT + compile + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + + + **/*Test.java + + + + + true + + + + + + + \ No newline at end of file diff --git a/tdenginereader/src/main/assembly/package.xml b/tdenginereader/src/main/assembly/package.xml new file mode 100755 index 0000000..b52f20f --- /dev/null +++ b/tdenginereader/src/main/assembly/package.xml @@ -0,0 +1,34 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/tdenginereader + + + target/ + + tdenginereader-0.0.1-SNAPSHOT.jar + + plugin/reader/tdenginereader + + + + + + false + plugin/reader/tdenginereader/libs + runtime + + + diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java new file mode 100644 index 0000000..eddf98e --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java @@ -0,0 +1,19 @@ +package com.alibaba.datax.plugin.reader; + +public class Key { + + public static final String JDBC_URL = "jdbcUrl"; +// public static final String HOST = "host"; +// public static final String PORT = "port"; +// public static final String DB = "db"; + public static final String TABLE = "table"; + public static final String USER = "user"; + public static final String PASSWORD = "password"; + public static final String CONNECTION = "connection"; +// public static final String SQL = "sql"; + public static final String BEGIN_DATETIME = "beginDateTime"; + public static final String END_DATETIME = "endDateTime"; + public static final String SPLIT_INTERVAL = "splitInterval"; + public static final String COLUMN = "column"; + public static final String MANDATORY_ENCODING = "mandatoryEncoding"; +} diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java new file mode 100644 index 0000000..dfdce7b --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -0,0 +1,291 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.constant.CommonConstant; +import com.alibaba.datax.common.element.*; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; +import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TDengineReader extends Reader { + private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + public static class Job extends Reader.Job { + + private static final Logger LOG = LoggerFactory.getLogger(Job.class); + private Configuration originalConfig; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + // check user + String user = this.originalConfig.getString(Key.USER); + if (StringUtils.isBlank(user)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.USER + "] is not set."); + + // check password + String password = this.originalConfig.getString(Key.PASSWORD); + if (StringUtils.isBlank(password)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.PASSWORD + "] is not set."); + + SimpleDateFormat format = new SimpleDateFormat(DATETIME_FORMAT); + // check beginDateTime + String beginDatetime = this.originalConfig.getString(Key.BEGIN_DATETIME); + if (StringUtils.isBlank(beginDatetime)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] is not set."); + Long start; + try { + start = format.parse(beginDatetime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); + } + + // check endDateTime + String endDatetime = this.originalConfig.getString(Key.END_DATETIME); + if (StringUtils.isBlank(endDatetime)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.END_DATETIME + "] is not set."); + Long end; + try { + end = format.parse(endDatetime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.END_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); + } + if (start >= end) + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] should be less than the parameter [" + Key.END_DATETIME + "]."); + + // check splitInterval + String splitInterval = this.originalConfig.getString(Key.SPLIT_INTERVAL); + Long split; + if (StringUtils.isBlank(splitInterval)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.SPLIT_INTERVAL + "] is not set."); + try { + split = parseSplitInterval(splitInterval); + } catch (Exception e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.SPLIT_INTERVAL + "] should be like: \"123d|h|m|s\", error: " + e.getMessage()); + } + + this.originalConfig.set(Key.BEGIN_DATETIME, start); + this.originalConfig.set(Key.END_DATETIME, end); + this.originalConfig.set(Key.SPLIT_INTERVAL, split); + + // check connection + List connection = this.originalConfig.getList(Key.CONNECTION); + if (connection == null || connection.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set."); + for (int i = 0; i < connection.size(); i++) { + Configuration conn = Configuration.from(connection.get(i).toString()); + List table = conn.getList(Key.TABLE); + if (table == null || table.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.TABLE + "] of connection[" + (i + 1) + "] is not set."); + String jdbcUrl = conn.getString(Key.JDBC_URL); + if (StringUtils.isBlank(jdbcUrl)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.JDBC_URL + "] of connection[" + (i + 1) + "] is not set."); + } + + // check column + List column = this.originalConfig.getList(Key.COLUMN); + if (column == null || column.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set or is empty."); + } + + @Override + public void destroy() { + + } + + @Override + public List split(int adviceNumber) { + List configurations = new ArrayList<>(); + // do split + Long start = this.originalConfig.getLong(Key.BEGIN_DATETIME); + Long end = this.originalConfig.getLong(Key.END_DATETIME); + Long split = this.originalConfig.getLong(Key.SPLIT_INTERVAL); + + List conns = this.originalConfig.getList(Key.CONNECTION); + + for (Long ts = start; ts < end; ts += split) { + for (int i = 0; i < conns.size(); i++) { + Configuration clone = this.originalConfig.clone(); + clone.remove(Key.SPLIT_INTERVAL); + + clone.set(Key.BEGIN_DATETIME, ts); + clone.set(Key.END_DATETIME, Math.min(ts + split, end)); + + Configuration conf = Configuration.from(conns.get(i).toString()); + String jdbcUrl = conf.getString(Key.JDBC_URL); + clone.set(Key.JDBC_URL, jdbcUrl); + clone.set(Key.TABLE, conf.getList(Key.TABLE)); + + // 抽取 jdbcUrl 中的 ip/port 进行资源使用的打标，以提供给 core 做有意义的 shuffle 操作 + clone.set(CommonConstant.LOAD_BALANCE_RESOURCE_MARK, DataBaseType.parseIpFromJdbcUrl(jdbcUrl)); + clone.remove(Key.CONNECTION); + + configurations.add(clone); + LOG.info("Configuration: {}", JSON.toJSONString(clone)); + } + } + return configurations; + } + } + + public static class Task extends Reader.Task { + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + private Connection conn; + + private Long startTime; + private Long endTime; + private List tables; + private List columns; + private String mandatoryEncoding; + + @Override + public void init() { + Configuration readerSliceConfig = super.getPluginJobConf(); + LOG.info("getPluginJobConf: {}", JSON.toJSONString(readerSliceConfig)); + + String url = readerSliceConfig.getString(Key.JDBC_URL); + if (StringUtils.isBlank(url)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.JDBC_URL + "] is not set."); + + tables = readerSliceConfig.getList(Key.TABLE, String.class); + columns = readerSliceConfig.getList(Key.COLUMN, String.class); + + String user = readerSliceConfig.getString(Key.USER); + String password = readerSliceConfig.getString(Key.PASSWORD); + + try { + conn = DriverManager.getConnection(url, user, password); + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.CONNECTION_FAILED, + "The parameter [" + Key.JDBC_URL + "] : " + url + " failed to connect since: " + e.getMessage()); + } + + this.mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); + + startTime = readerSliceConfig.getLong(Key.BEGIN_DATETIME); + endTime = readerSliceConfig.getLong(Key.END_DATETIME); + } + + + @Override + public void destroy() { + + } + + @Override + public void startRead(RecordSender recordSender) { + try (Statement stmt = conn.createStatement()) { + for (int i = 0; i < tables.size(); i++) { + String sql = "select " + StringUtils.join(columns, ",") + " from " + tables.get(i) + " where _c0 >= " + startTime + " and _c0 < " + endTime; + ResultSet rs = stmt.executeQuery(sql); + ResultSetMetaData metaData = rs.getMetaData(); + while (rs.next()) { + transportOneRecord(recordSender, rs, metaData, metaData.getColumnCount(), this.mandatoryEncoding); + } + } + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + } finally { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + private Record transportOneRecord(RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, int columnCount, String mandatoryEncoding) { + Record record = buildRecord(recordSender, rs, metaData, columnCount, mandatoryEncoding); + recordSender.sendToWriter(record); + return record; + } + + private Record buildRecord(RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, int columnCount, String mandatoryEncoding) { + Record record = recordSender.createRecord(); + + try { + for (int i = 1; i <= columnCount; i++) { + int columnType = metaData.getColumnType(i); + switch (columnType) { + case Types.SMALLINT: + case Types.TINYINT: + case Types.INTEGER: + case Types.BIGINT: + record.addColumn(new LongColumn(rs.getString(i))); + break; + case Types.FLOAT: + case Types.DOUBLE: + record.addColumn(new DoubleColumn(rs.getString(i))); + break; + case Types.BOOLEAN: + record.addColumn(new BoolColumn(rs.getBoolean(i))); + break; + case Types.TIMESTAMP: + record.addColumn(new DateColumn(rs.getTimestamp(i))); + break; + case Types.BINARY: + record.addColumn(new BytesColumn(rs.getBytes(i))); + break; + case Types.NCHAR: + String rawData; + if (StringUtils.isBlank(mandatoryEncoding)) { + rawData = rs.getString(i); + } else { + rawData = new String((rs.getBytes(i) == null ? new byte[0] : rs.getBytes(i)), mandatoryEncoding); + } + record.addColumn(new StringColumn(rawData)); + break; + } + } + } catch (SQLException | UnsupportedEncodingException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + } + return record; + } + } + + private static final long second = 1000; + private static final long minute = 60 * second; + private static final long hour = 60 * minute; + private static final long day = 24 * hour; + + private static Long parseSplitInterval(String splitInterval) throws Exception { + Pattern compile = Pattern.compile("^(\\d+)([dhms])$"); + Matcher matcher = compile.matcher(splitInterval); + while (matcher.find()) { + Long value = Long.valueOf(matcher.group(1)); + if (value == 0) + throw new Exception("invalid splitInterval: 0"); + char unit = matcher.group(2).charAt(0); + switch (unit) { + case 'd': + return value * day; + default: + case 'h': + return value * hour; + case 'm': + return value * minute; + case 's': + return value * second; + } + } + throw new Exception("invalid splitInterval: " + splitInterval); + } + +} diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java new file mode 100644 index 0000000..68bc11e --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java @@ -0,0 +1,33 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum TDengineReaderErrorCode implements ErrorCode { + + REQUIRED_VALUE("TDengineReader-00", "缺失必要的值"), + ILLEGAL_VALUE("TDengineReader-01", "值非法"), + CONNECTION_FAILED("TDengineReader-02", "连接错误"); + + private final String code; + private final String description; + + TDengineReaderErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); + } +} diff --git a/tdenginereader/src/main/resources/plugin.json b/tdenginereader/src/main/resources/plugin.json new file mode 100755 index 0000000..dc91982 --- /dev/null +++ b/tdenginereader/src/main/resources/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "tdenginereader", + "class": "com.alibaba.datax.plugin.reader.TDengineReader", + "description": { + "useScene": "data migration from tdengine", + "mechanism": "use JNI to read data from tdengine." + }, + "developer": "zyyang-taosdata" +} \ No newline at end of file diff --git a/tdenginereader/src/main/resources/plugin_job_template.json b/tdenginereader/src/main/resources/plugin_job_template.json new file mode 100644 index 0000000..3e09dff --- /dev/null +++ b/tdenginereader/src/main/resources/plugin_job_template.json @@ -0,0 +1,14 @@ +{ + "name": "tdenginereader", + "parameter": { + "host": "127.0.0.1", + "port": 6030, + "db": "test", + "user": "root", + "password": "taosdata", + "sql": "select * from weather", + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-02 00:00:00", + "splitInterval": "1h" + } +} \ No newline at end of file diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java new file mode 100644 index 0000000..1445309 --- /dev/null +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java @@ -0,0 +1,47 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.util.Configuration; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.stream.IntStream; + +public class TDengineReaderTest { + + TDengineReader.Job job; + + @Before + public void before() { + job = new TDengineReader.Job(); + Configuration configuration = Configuration.from("{" + + "\"user\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"beginDateTime\": \"2021-01-01 00:00:00\"," + + "\"endDateTime\": \"2021-01-01 10:00:00\"," + + "\"splitInterval\": \"1h\"" + + "}"); + job.setPluginJobConf(configuration); + } + + @Test + public void jobInit() { + job.init(); + + // assert + Configuration conf = job.getPluginJobConf(); + Assert.assertEquals("select * from weather", conf.getString("sql")); + } + + @Test + public void jobSplit() { + // when + job.init(); + List configurationList = job.split(1); + +// IntStream.range(0, configurationList.size()).forEach(i -> System.out.println(i + ": " + configurationList.get(i))); + } + +} \ No newline at end of file From 86553c490dded5709c047de33c1f0cda9004a18d Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 16 Dec 2021 11:36:44 +0800 Subject: [PATCH 068/114] [TD-10786]: tdenginereader implementation --- package.xml | 7 + pom.xml | 1 + tdenginereader/doc/tdenginereader.md | 162 ++++++++++ tdenginereader/pom.xml | 101 ++++++ tdenginereader/src/main/assembly/package.xml | 34 ++ .../com/alibaba/datax/plugin/reader/Key.java | 19 ++ .../datax/plugin/reader/TDengineReader.java | 291 ++++++++++++++++++ .../reader/TDengineReaderErrorCode.java | 33 ++ tdenginereader/src/main/resources/plugin.json | 9 + .../main/resources/plugin_job_template.json | 21 ++ .../plugin/reader/TDengineReaderTest.java | 73 +++++ 11 files changed, 751 insertions(+) create mode 100644 tdenginereader/doc/tdenginereader.md create mode 100644 tdenginereader/pom.xml create mode 100755 tdenginereader/src/main/assembly/package.xml create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java create mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java create mode 100755 tdenginereader/src/main/resources/plugin.json create mode 100644 tdenginereader/src/main/resources/plugin_job_template.json create mode 100644 tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java diff --git a/package.xml b/package.xml index d95feb4..3f024f9 100755 --- a/package.xml +++ b/package.xml @@ -180,6 +180,13 @@ datax + + tdenginereader/target/datax/ + + **/*.* + + datax + diff --git a/pom.xml b/pom.xml index 1a9da81..7224113 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,7 @@ hbase20xsqlreader hbase20xsqlwriter kuduwriter + tdenginereader diff --git a/tdenginereader/doc/tdenginereader.md b/tdenginereader/doc/tdenginereader.md new file mode 100644 index 0000000..3c683a6 --- /dev/null +++ b/tdenginereader/doc/tdenginereader.md @@ -0,0 +1,162 @@ +# DataX TDengineReader + +## 1 快速介绍 + +TDengineReader 插件实现了 TDengine 读取数据的功能。 + +## 2 实现原理 + +TDengineReader 通过TDengine的JDBC driver查询获取数据。 + +## 3 功能说明 + +### 3.1 配置样例 + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "user": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "meters" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "ts", + "current", + "voltage", + "phase" + ], + "beginDateTime": "2017-07-14 10:40:00", + "endDateTime": "2017-08-14 10:40:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + +### 3.2 参数说明 + +* **username** + * 描述：TDengine实例的用户名
+ * 必选：是
+ * 默认值：无
+* **password** + * 描述：TDengine实例的密码
+ * 必选：是
+ * 默认值：无
+* **table** + * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构， + TDengineReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
+ * 必选：是
+ * 默认值：无
+* **jdbcUrl** + * 描述：TDengine数据库的JDBC连接信息。注意，jdbcUrl必须包含在connection配置单元中。JdbcUrl具体请参看TDengine官方文档。 + * 必选：是
+ * 默认值：无
+* **beginDateTime** + * 描述：数据的开始时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss
+ * 必选：是
+ * 默认值：无
+* **endDateTime** + * 描述：数据的结束时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss
+ * 必选：是
+ * 默认值：无
+* **splitInterval** + * 描述：按照splitInterval来划分task, 每splitInterval创建一个task。例如，20d代表按照每20天的数据划分为1个task。 + 可以配置的时间单位为：d（天）, h（小时）, m（分钟）, s（秒）
+ * 必选：是
+ * 默认值：无
+ +### 3.3 类型转换 +| TDengine 数据类型 | DataX 内部类型 | +| --------------- | ------------- | +| TINYINT | Long | +| SMALLINT | Long | +| INTEGER | Long | +| BIGINT | Long | +| FLOAT | Double | +| DOUBLE | Double | +| BOOLEAN | Bool | +| TIMESTAMP | Date | +| BINARY | Bytes | +| NCHAR | String | + + +## 4 性能报告 + +### 4.1 环境准备 + +#### 4.1.1 数据特征 + +建表语句： + +单行记录类似于： + +#### 4.1.2 机器参数 + +* 执行DataX的机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: DataX 数据不落磁盘，不统计此项 + +* TDengine数据库机器参数为: + 1. cpu: + 2. mem: + 3. net: 千兆双网卡 + 4. disc: + +#### 4.1.3 DataX jvm 参数 + + -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + +### 4.2 测试报告 + +#### 4.2.1 单表测试报告 + +| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡流出流量(MB/s)|DataX机器运行负载|DB网卡进入流量(MB/s)|DB运行负载|DB TPS| +|--------| --------|--------|--------|--------|--------|--------|--------| +|1| | | | | | | | +|4| | | | | | | | +|8| | | | | | | | +|16| | | | | | | | +|32| | | | | | | | + +说明： + +1. 这里的单表，主键类型为 bigint(20),自增。 +2. batchSize 和 通道个数，对性能影响较大。 + +#### 4.2.4 性能测试小结 + +1. +2. + +## 5 约束限制 + +## FAQ \ No newline at end of file diff --git a/tdenginereader/pom.xml b/tdenginereader/pom.xml new file mode 100644 index 0000000..8c0f664 --- /dev/null +++ b/tdenginereader/pom.xml @@ -0,0 +1,101 @@ + + + + datax-all + com.alibaba.datax + 0.0.1-SNAPSHOT + + 4.0.0 + + tdenginereader + + + 8 + 8 + + + + + com.alibaba.datax + datax-common + ${datax-project-version} + + + slf4j-log4j12 + org.slf4j + + + + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.34 + + + + junit + junit + ${junit-version} + test + + + com.alibaba.datax + plugin-rdbms-util + 0.0.1-SNAPSHOT + compile + + + + + + + + maven-compiler-plugin + + ${jdk-version} + ${jdk-version} + ${project-sourceEncoding} + + + + maven-assembly-plugin + + + src/main/assembly/package.xml + + datax + + + + dwzip + package + + single + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + + + **/*Test.java + + + + + true + + + + + + + \ No newline at end of file diff --git a/tdenginereader/src/main/assembly/package.xml b/tdenginereader/src/main/assembly/package.xml new file mode 100755 index 0000000..b52f20f --- /dev/null +++ b/tdenginereader/src/main/assembly/package.xml @@ -0,0 +1,34 @@ + + + + dir + + false + + + src/main/resources + + plugin.json + plugin_job_template.json + + plugin/reader/tdenginereader + + + target/ + + tdenginereader-0.0.1-SNAPSHOT.jar + + plugin/reader/tdenginereader + + + + + + false + plugin/reader/tdenginereader/libs + runtime + + + diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java new file mode 100644 index 0000000..eddf98e --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java @@ -0,0 +1,19 @@ +package com.alibaba.datax.plugin.reader; + +public class Key { + + public static final String JDBC_URL = "jdbcUrl"; +// public static final String HOST = "host"; +// public static final String PORT = "port"; +// public static final String DB = "db"; + public static final String TABLE = "table"; + public static final String USER = "user"; + public static final String PASSWORD = "password"; + public static final String CONNECTION = "connection"; +// public static final String SQL = "sql"; + public static final String BEGIN_DATETIME = "beginDateTime"; + public static final String END_DATETIME = "endDateTime"; + public static final String SPLIT_INTERVAL = "splitInterval"; + public static final String COLUMN = "column"; + public static final String MANDATORY_ENCODING = "mandatoryEncoding"; +} diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java new file mode 100644 index 0000000..dfdce7b --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -0,0 +1,291 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.constant.CommonConstant; +import com.alibaba.datax.common.element.*; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordSender; +import com.alibaba.datax.common.spi.Reader; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; +import com.alibaba.fastjson.JSON; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.UnsupportedEncodingException; +import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TDengineReader extends Reader { + private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + public static class Job extends Reader.Job { + + private static final Logger LOG = LoggerFactory.getLogger(Job.class); + private Configuration originalConfig; + + @Override + public void init() { + this.originalConfig = super.getPluginJobConf(); + // check user + String user = this.originalConfig.getString(Key.USER); + if (StringUtils.isBlank(user)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.USER + "] is not set."); + + // check password + String password = this.originalConfig.getString(Key.PASSWORD); + if (StringUtils.isBlank(password)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.PASSWORD + "] is not set."); + + SimpleDateFormat format = new SimpleDateFormat(DATETIME_FORMAT); + // check beginDateTime + String beginDatetime = this.originalConfig.getString(Key.BEGIN_DATETIME); + if (StringUtils.isBlank(beginDatetime)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] is not set."); + Long start; + try { + start = format.parse(beginDatetime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); + } + + // check endDateTime + String endDatetime = this.originalConfig.getString(Key.END_DATETIME); + if (StringUtils.isBlank(endDatetime)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.END_DATETIME + "] is not set."); + Long end; + try { + end = format.parse(endDatetime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.END_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); + } + if (start >= end) + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] should be less than the parameter [" + Key.END_DATETIME + "]."); + + // check splitInterval + String splitInterval = this.originalConfig.getString(Key.SPLIT_INTERVAL); + Long split; + if (StringUtils.isBlank(splitInterval)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.SPLIT_INTERVAL + "] is not set."); + try { + split = parseSplitInterval(splitInterval); + } catch (Exception e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.SPLIT_INTERVAL + "] should be like: \"123d|h|m|s\", error: " + e.getMessage()); + } + + this.originalConfig.set(Key.BEGIN_DATETIME, start); + this.originalConfig.set(Key.END_DATETIME, end); + this.originalConfig.set(Key.SPLIT_INTERVAL, split); + + // check connection + List connection = this.originalConfig.getList(Key.CONNECTION); + if (connection == null || connection.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set."); + for (int i = 0; i < connection.size(); i++) { + Configuration conn = Configuration.from(connection.get(i).toString()); + List table = conn.getList(Key.TABLE); + if (table == null || table.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.TABLE + "] of connection[" + (i + 1) + "] is not set."); + String jdbcUrl = conn.getString(Key.JDBC_URL); + if (StringUtils.isBlank(jdbcUrl)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.JDBC_URL + "] of connection[" + (i + 1) + "] is not set."); + } + + // check column + List column = this.originalConfig.getList(Key.COLUMN); + if (column == null || column.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set or is empty."); + } + + @Override + public void destroy() { + + } + + @Override + public List split(int adviceNumber) { + List configurations = new ArrayList<>(); + // do split + Long start = this.originalConfig.getLong(Key.BEGIN_DATETIME); + Long end = this.originalConfig.getLong(Key.END_DATETIME); + Long split = this.originalConfig.getLong(Key.SPLIT_INTERVAL); + + List conns = this.originalConfig.getList(Key.CONNECTION); + + for (Long ts = start; ts < end; ts += split) { + for (int i = 0; i < conns.size(); i++) { + Configuration clone = this.originalConfig.clone(); + clone.remove(Key.SPLIT_INTERVAL); + + clone.set(Key.BEGIN_DATETIME, ts); + clone.set(Key.END_DATETIME, Math.min(ts + split, end)); + + Configuration conf = Configuration.from(conns.get(i).toString()); + String jdbcUrl = conf.getString(Key.JDBC_URL); + clone.set(Key.JDBC_URL, jdbcUrl); + clone.set(Key.TABLE, conf.getList(Key.TABLE)); + + // 抽取 jdbcUrl 中的 ip/port 进行资源使用的打标，以提供给 core 做有意义的 shuffle 操作 + clone.set(CommonConstant.LOAD_BALANCE_RESOURCE_MARK, DataBaseType.parseIpFromJdbcUrl(jdbcUrl)); + clone.remove(Key.CONNECTION); + + configurations.add(clone); + LOG.info("Configuration: {}", JSON.toJSONString(clone)); + } + } + return configurations; + } + } + + public static class Task extends Reader.Task { + private static final Logger LOG = LoggerFactory.getLogger(Task.class); + private Connection conn; + + private Long startTime; + private Long endTime; + private List tables; + private List columns; + private String mandatoryEncoding; + + @Override + public void init() { + Configuration readerSliceConfig = super.getPluginJobConf(); + LOG.info("getPluginJobConf: {}", JSON.toJSONString(readerSliceConfig)); + + String url = readerSliceConfig.getString(Key.JDBC_URL); + if (StringUtils.isBlank(url)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.JDBC_URL + "] is not set."); + + tables = readerSliceConfig.getList(Key.TABLE, String.class); + columns = readerSliceConfig.getList(Key.COLUMN, String.class); + + String user = readerSliceConfig.getString(Key.USER); + String password = readerSliceConfig.getString(Key.PASSWORD); + + try { + conn = DriverManager.getConnection(url, user, password); + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.CONNECTION_FAILED, + "The parameter [" + Key.JDBC_URL + "] : " + url + " failed to connect since: " + e.getMessage()); + } + + this.mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); + + startTime = readerSliceConfig.getLong(Key.BEGIN_DATETIME); + endTime = readerSliceConfig.getLong(Key.END_DATETIME); + } + + + @Override + public void destroy() { + + } + + @Override + public void startRead(RecordSender recordSender) { + try (Statement stmt = conn.createStatement()) { + for (int i = 0; i < tables.size(); i++) { + String sql = "select " + StringUtils.join(columns, ",") + " from " + tables.get(i) + " where _c0 >= " + startTime + " and _c0 < " + endTime; + ResultSet rs = stmt.executeQuery(sql); + ResultSetMetaData metaData = rs.getMetaData(); + while (rs.next()) { + transportOneRecord(recordSender, rs, metaData, metaData.getColumnCount(), this.mandatoryEncoding); + } + } + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + } finally { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + private Record transportOneRecord(RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, int columnCount, String mandatoryEncoding) { + Record record = buildRecord(recordSender, rs, metaData, columnCount, mandatoryEncoding); + recordSender.sendToWriter(record); + return record; + } + + private Record buildRecord(RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, int columnCount, String mandatoryEncoding) { + Record record = recordSender.createRecord(); + + try { + for (int i = 1; i <= columnCount; i++) { + int columnType = metaData.getColumnType(i); + switch (columnType) { + case Types.SMALLINT: + case Types.TINYINT: + case Types.INTEGER: + case Types.BIGINT: + record.addColumn(new LongColumn(rs.getString(i))); + break; + case Types.FLOAT: + case Types.DOUBLE: + record.addColumn(new DoubleColumn(rs.getString(i))); + break; + case Types.BOOLEAN: + record.addColumn(new BoolColumn(rs.getBoolean(i))); + break; + case Types.TIMESTAMP: + record.addColumn(new DateColumn(rs.getTimestamp(i))); + break; + case Types.BINARY: + record.addColumn(new BytesColumn(rs.getBytes(i))); + break; + case Types.NCHAR: + String rawData; + if (StringUtils.isBlank(mandatoryEncoding)) { + rawData = rs.getString(i); + } else { + rawData = new String((rs.getBytes(i) == null ? new byte[0] : rs.getBytes(i)), mandatoryEncoding); + } + record.addColumn(new StringColumn(rawData)); + break; + } + } + } catch (SQLException | UnsupportedEncodingException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + } + return record; + } + } + + private static final long second = 1000; + private static final long minute = 60 * second; + private static final long hour = 60 * minute; + private static final long day = 24 * hour; + + private static Long parseSplitInterval(String splitInterval) throws Exception { + Pattern compile = Pattern.compile("^(\\d+)([dhms])$"); + Matcher matcher = compile.matcher(splitInterval); + while (matcher.find()) { + Long value = Long.valueOf(matcher.group(1)); + if (value == 0) + throw new Exception("invalid splitInterval: 0"); + char unit = matcher.group(2).charAt(0); + switch (unit) { + case 'd': + return value * day; + default: + case 'h': + return value * hour; + case 'm': + return value * minute; + case 's': + return value * second; + } + } + throw new Exception("invalid splitInterval: " + splitInterval); + } + +} diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java new file mode 100644 index 0000000..68bc11e --- /dev/null +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java @@ -0,0 +1,33 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.spi.ErrorCode; + +public enum TDengineReaderErrorCode implements ErrorCode { + + REQUIRED_VALUE("TDengineReader-00", "缺失必要的值"), + ILLEGAL_VALUE("TDengineReader-01", "值非法"), + CONNECTION_FAILED("TDengineReader-02", "连接错误"); + + private final String code; + private final String description; + + TDengineReaderErrorCode(String code, String description) { + this.code = code; + this.description = description; + } + + @Override + public String getCode() { + return this.code; + } + + @Override + public String getDescription() { + return this.description; + } + + @Override + public String toString() { + return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); + } +} diff --git a/tdenginereader/src/main/resources/plugin.json b/tdenginereader/src/main/resources/plugin.json new file mode 100755 index 0000000..7ccdbe6 --- /dev/null +++ b/tdenginereader/src/main/resources/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "tdenginereader", + "class": "com.alibaba.datax.plugin.reader.TDengineReader", + "description": { + "useScene": "data migration from tdengine", + "mechanism": "use JDBC to read data from tdengine." + }, + "developer": "zyyang-taosdata" +} \ No newline at end of file diff --git a/tdenginereader/src/main/resources/plugin_job_template.json b/tdenginereader/src/main/resources/plugin_job_template.json new file mode 100644 index 0000000..2c7f4cb --- /dev/null +++ b/tdenginereader/src/main/resources/plugin_job_template.json @@ -0,0 +1,21 @@ +{ + "name": "tdenginereader", + "parameter": { + "user": "", + "password": "", + "connection": [ + { + "table": [ + "" + ], + "jdbcUrl": "" + } + ], + "column": [ + "" + ], + "beginDateTime": "", + "endDateTime": "", + "splitInterval": "" + } +} \ No newline at end of file diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java new file mode 100644 index 0000000..9d83f74 --- /dev/null +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java @@ -0,0 +1,73 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.common.util.Configuration; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.List; +import java.util.stream.IntStream; + +public class TDengineReaderTest { + + TDengineReader.Job job; + + @Before + public void before() { + job = new TDengineReader.Job(); + Configuration configuration = Configuration.from("{" + + "\"user\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + + "\"beginDateTime\": \"2021-01-01 00:00:00\"," + + "\"endDateTime\": \"2021-01-01 12:00:00\"," + + "\"splitInterval\": \"1h\"" + + "}"); + job.setPluginJobConf(configuration); + } + + @Test + public void jobInit() throws ParseException { + // when + job.init(); + + // assert + Configuration conf = job.getPluginJobConf(); + + Assert.assertEquals("root", conf.getString("user")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("weather", conf.getString("connection[0].table[0]")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl")); + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + Long start = sdf.parse("2021-01-01 00:00:00").getTime(); + Assert.assertEquals(start, conf.getLong("beginDateTime")); + + Long end = sdf.parse("2021-01-01 12:00:00").getTime(); + Assert.assertEquals(end, conf.getLong("endDateTime")); + + Assert.assertEquals(new Long(3600 * 1000), conf.getLong("splitInterval")); + } + + @Test + public void jobSplit() { + // when + job.init(); + List configurationList = job.split(1); + + // assert + Assert.assertEquals(12, configurationList.size()); + for (int i = 0; i < configurationList.size(); i++) { + Configuration conf = configurationList.get(i); + Assert.assertEquals("root", conf.getString("user")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("weather", conf.getString("table[0]")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); + } + } + +} \ No newline at end of file From 61dc03e9f8cdcf215ac739535c24c417f6b8215b Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 16 Dec 2021 17:00:08 +0800 Subject: [PATCH 069/114] change user to username --- tdenginewriter/doc/tdenginewriter-CN.md | 10 +++++----- tdenginewriter/doc/tdenginewriter.md | 10 +++++----- .../datax/plugin/writer/tdenginewriter/Key.java | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md index c0ce768..ac5a555 100644 --- a/tdenginewriter/doc/tdenginewriter-CN.md +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -42,7 +42,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "host": "192.168.1.180", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata" } } @@ -63,7 +63,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea | --------- | -------------------- | -------- | -------- | | host | TDengine实例的host | 是 | 无 | | port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | +| username | TDengine实例的用户名 | 否 | root | | password | TDengine实例的密码 | 否 | taosdata | | dbName | 目的数据库的名称 | 是 | 无 | | batchSize | 每次批量插入多少记录 | 否 | 1 | @@ -142,7 +142,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "host": "localhost", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "stable": "stock", "tagColumn": { @@ -174,7 +174,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea | --------------- | -------------------- | ---------------- | -------- | ------------------ | | host | TDengine实例的host | 是 | 无 | | port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | +| username | TDengine实例的用户名 | 否 | root | | password | TDengine实例的密码 | 否 | taosdata | | dbName | 目的数据库的名称 | 是 | 无 | | batchSize | 每次批量插入多少记录 | 否 | 1000 | @@ -290,7 +290,7 @@ CREATE TABLE IF NOT EXISTS weather( "host": "127.0.0.1", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "batchSize": 1000, "stable": "weather", diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 1e7fb64..fb8cf64 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -39,7 +39,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s "host": "192.168.1.180", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata" } } @@ -60,7 +60,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s | --------- | ------------------------------ | -------- | -------- | | host | host of TDengine | Yes | | | port | port of TDengine | Yes | | -| user | use name of TDengine | No | root | +| username | use name of TDengine | No | root | | password | password of TDengine | No | taosdata | | dbName | name of target database | No | | | batchSize | batch size of insert operation | No | 1 | @@ -137,7 +137,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s "host": "localhost", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "stable": "stock", "tagColumn": { @@ -170,7 +170,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s | --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | | host | host ofTDengine | Yes | | | port | port of TDengine | Yes | | -| user | user name of TDengine | No | root | +| username | username of TDengine | No | root | | password | password of TDengine | No | taosdata | | dbName | name of target database | Yes | | | batchSize | batch size of insert operation | No | 1000 | @@ -287,7 +287,7 @@ CREATE TABLE IF NOT EXISTS weather( "host": "127.0.0.1", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "batchSize": 1000, "stable": "weather", diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index a061e97..7fb383e 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -4,7 +4,7 @@ public class Key { public static final String HOST = "host"; public static final String PORT = "port"; public static final String DBNAME = "dbName"; - public static final String USER = "user"; + public static final String USER = "username"; public static final String PASSWORD = "password"; public static final String BATCH_SIZE = "batchSize"; public static final String STABLE = "stable"; From c7fc73bcb647c953a6622a87a40996e243cc2b79 Mon Sep 17 00:00:00 2001 From: dingbo Date: Thu, 16 Dec 2021 17:00:08 +0800 Subject: [PATCH 070/114] change user to username --- tdenginewriter/doc/tdenginewriter-CN.md | 10 +++++----- tdenginewriter/doc/tdenginewriter.md | 10 +++++----- .../datax/plugin/writer/tdenginewriter/Key.java | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md index abd92de..a868f8f 100644 --- a/tdenginewriter/doc/tdenginewriter-CN.md +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -42,7 +42,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "host": "192.168.1.180", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata" } } @@ -63,7 +63,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea | --------- | -------------------- | -------- | -------- | | host | TDengine实例的host | 是 | 无 | | port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | +| username | TDengine实例的用户名 | 否 | root | | password | TDengine实例的密码 | 否 | taosdata | | dbName | 目的数据库的名称 | 是 | 无 | | batchSize | 每次批量插入多少记录 | 否 | 1 | @@ -142,7 +142,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "host": "localhost", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "stable": "stock", "tagColumn": { @@ -174,7 +174,7 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea | --------------- | -------------------- | ---------------- | -------- | ------------------ | | host | TDengine实例的host | 是 | 无 | | port | TDengine实例的port | 是 | 无 | -| user | TDengine实例的用户名 | 否 | root | +| username | TDengine实例的用户名 | 否 | root | | password | TDengine实例的密码 | 否 | taosdata | | dbName | 目的数据库的名称 | 是 | 无 | | batchSize | 每次批量插入多少记录 | 否 | 1000 | @@ -290,7 +290,7 @@ CREATE TABLE IF NOT EXISTS weather( "host": "127.0.0.1", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "batchSize": 1000, "stable": "weather", diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index fd19057..6f19c11 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -39,7 +39,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s "host": "192.168.1.180", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata" } } @@ -60,7 +60,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s | --------- | ------------------------------ | -------- | -------- | | host | host of TDengine | Yes | | | port | port of TDengine | Yes | | -| user | use name of TDengine | No | root | +| username | use name of TDengine | No | root | | password | password of TDengine | No | taosdata | | dbName | name of target database | No | | | batchSize | batch size of insert operation | No | 1 | @@ -137,7 +137,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s "host": "localhost", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "stable": "stock", "tagColumn": { @@ -170,7 +170,7 @@ TDengineWriter get records from DataX Framework that are generated from reader s | --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | | host | host ofTDengine | Yes | | | port | port of TDengine | Yes | | -| user | user name of TDengine | No | root | +| username | username of TDengine | No | root | | password | password of TDengine | No | taosdata | | dbName | name of target database | Yes | | | batchSize | batch size of insert operation | No | 1000 | @@ -287,7 +287,7 @@ CREATE TABLE IF NOT EXISTS weather( "host": "127.0.0.1", "port": 6030, "dbName": "test", - "user": "root", + "username": "root", "password": "taosdata", "batchSize": 1000, "stable": "weather", diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index a061e97..7fb383e 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -4,7 +4,7 @@ public class Key { public static final String HOST = "host"; public static final String PORT = "port"; public static final String DBNAME = "dbName"; - public static final String USER = "user"; + public static final String USER = "username"; public static final String PASSWORD = "password"; public static final String BATCH_SIZE = "batchSize"; public static final String STABLE = "stable"; From 69320e750a55c4cee51ea83b015feb11a73741f0 Mon Sep 17 00:00:00 2001 From: dingbo Date: Wed, 22 Dec 2021 10:08:00 +0800 Subject: [PATCH 071/114] rename resource key --- tdenginewriter/src/main/resources/tdenginewritermsg.properties | 2 +- .../src/main/resources/tdenginewritermsg_en_US.properties | 2 +- .../src/main/resources/tdenginewritermsg_zh_CN.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg.properties b/tdenginewriter/src/main/resources/tdenginewritermsg.properties index 4aaa220..48d862e 100644 --- a/tdenginewriter/src/main/resources/tdenginewritermsg.properties +++ b/tdenginewriter/src/main/resources/tdenginewritermsg.properties @@ -1,4 +1,4 @@ -try_get_schema_fromdb=fail to get structure info of target table from configure file and will try to get it from database +try_get_schema_from_db=fail to get structure info of target table from configure file and will try to get it from database batch_size_too_small='batchSize' is too small, please increase it and try again column_number_error=number of columns is less than expected tag_value_error=tag columns include 'null' value diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties index 4aaa220..48d862e 100644 --- a/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties +++ b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties @@ -1,4 +1,4 @@ -try_get_schema_fromdb=fail to get structure info of target table from configure file and will try to get it from database +try_get_schema_from_db=fail to get structure info of target table from configure file and will try to get it from database batch_size_too_small='batchSize' is too small, please increase it and try again column_number_error=number of columns is less than expected tag_value_error=tag columns include 'null' value diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties index 4b9552f..38d2cbb 100644 --- a/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties +++ b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties @@ -1,4 +1,4 @@ -try_get_schema_fromdb=\u65e0\u6cd5\u4ece\u914d\u7f6e\u6587\u4ef6\u83b7\u53d6\u8868\u7ed3\u6784\u4fe1\u606f\uff0c\u5c1d\u8bd5\u4ece\u6570\u636e\u5e93\u83b7\u53d6 +try_get_schema_from_db=\u65e0\u6cd5\u4ece\u914d\u7f6e\u6587\u4ef6\u83b7\u53d6\u8868\u7ed3\u6784\u4fe1\u606f\uff0c\u5c1d\u8bd5\u4ece\u6570\u636e\u5e93\u83b7\u53d6 batch_size_too_small=batchSize\u592a\u5c0f\uff0c\u4f1a\u589e\u52a0\u81ea\u52a8\u7c7b\u578b\u63a8\u65ad\u9519\u8bef\u7684\u6982\u7387\uff0c\u5efa\u8bae\u6539\u5927\u540e\u91cd\u8bd5 column_number_error=\u5b9e\u9645\u5217\u6570\u5c0f\u4e8e\u671f\u671b\u5217\u6570 tag_value_error=\u6807\u7b7e\u5217\u5305\u542bnull From ca39f422d0f78f44afd2178ceb17930098661d45 Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Wed, 22 Dec 2021 16:14:41 +0800 Subject: [PATCH 072/114] sd --- .../oceanbasev10reader/OceanBaseReader.java | 19 ++++- .../oceanbasev10reader/ext/ReaderJob.java | 31 ++++++++- .../util/DatabaseKeywordTransformer.java | 69 +++++++++++++++++++ .../util/ObReaderUtils.java | 10 +++ 4 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java index 0a4934a..db9d34e 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java @@ -1,8 +1,12 @@ package com.alibaba.datax.plugin.reader.oceanbasev10reader; +import java.sql.Array; import java.sql.Connection; +import java.util.ArrayList; import java.util.List; +import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.DatabaseKeywordTransformer; +import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +26,7 @@ public class OceanBaseReader extends Reader { public static class Job extends Reader.Job { private Configuration originalConfig = null; private ReaderJob readerJob; + private DataBaseType DATABASE_TYPE; private static final Logger LOG = LoggerFactory.getLogger(Task.class); @Override @@ -37,10 +42,15 @@ public class OceanBaseReader extends Reader { setDatabaseType(originalConfig); - this.readerJob = new ReaderJob(); - this.readerJob.init(this.originalConfig); - } + + this.readerJob = new ReaderJob(); + this.readerJob.init(this.originalConfig,DATABASE_TYPE); + } + @Override + public void prepare(){ + //ObReaderUtils.DATABASE_TYPE获取当前数据库的语法模式 + } @Override public void preCheck() { init(); @@ -90,6 +100,9 @@ public class OceanBaseReader extends Reader { } catch (Exception e){ LOG.warn("error in get compatible mode, using mysql as default: " + e.getMessage()); } + finally { + DATABASE_TYPE=ObReaderUtils.DATABASE_TYPE; + } } } diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java index c56155f..efeec2e 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java @@ -6,15 +6,44 @@ import com.alibaba.datax.common.constant.CommonConstant; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader; import com.alibaba.datax.plugin.rdbms.reader.Key; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; import com.alibaba.datax.plugin.rdbms.writer.Constant; +import com.alibaba.datax.plugin.reader.oceanbasev10reader.OceanBaseReader; +import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.DatabaseKeywordTransformer; import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.ObReaderUtils; +import com.alibaba.fastjson.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class ReaderJob extends CommonRdbmsReader.Job { public ReaderJob() { super(ObReaderUtils.DATABASE_TYPE); - } + } + public void init(Configuration originalConfig,DataBaseType databaseType){ + //将config中的column和table中的关键字进行转义 + final Logger LOG = LoggerFactory.getLogger(OceanBaseReader.Task.class); + try { + DatabaseKeywordTransformer.setDatabaseType(databaseType); + }catch (Exception e){ + LOG.warn("database type is "+databaseType+e.getMessage()); + } + List columns=originalConfig.getList(Key.COLUMN,String.class); + DatabaseKeywordTransformer.transferDatabaseKeywords(columns); + originalConfig.set(Key.COLUMN, columns); + + List conns=originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK,JSONObject.class); + for(int i=0;i tables=connConfig.getList(Key.TABLE,String.class); + DatabaseKeywordTransformer.transferDatabaseKeywords(tables); + originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK,i,Key.TABLE),tables); + } + super.init(originalConfig); + } @Override public List split(Configuration originalConfig, int adviceNumber) { List list = super.split(originalConfig, adviceNumber); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java new file mode 100644 index 0000000..ccbbfb2 --- /dev/null +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java @@ -0,0 +1,69 @@ +package com.alibaba.datax.plugin.reader.oceanbasev10reader.util; + +//java api +import java.util.HashSet; +import java.util.List; + +//dataX api +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; + +//该类用于转义数据库中的关键字 +public class DatabaseKeywordTransformer { + private static DataBaseType databaseType; + static HashSet databaseKeywords; + private static HashSet keywordsFromString2HashSet(final String Keywords){ + String[] keywordArray =Keywords.split(","); + HashSet res=new HashSet(); + for(String keyword: keywordArray){ + res.add(keyword); + } + return res; + } + public static void setDatabaseType(final DataBaseType databaseType) throws Exception { + if(databaseType==DatabaseKeywordTransformer.databaseType){ + return ; + } + DatabaseKeywordTransformer.databaseType = databaseType; + if(databaseType==DataBaseType.MySql){ + databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.MYSQL_KEYWORDS); + } + else if(databaseType==DataBaseType.Oracle){ + databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.ORACLE_KEYWORDS); + } + else if(databaseType==DataBaseType.SQLServer){ + databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.SQLSERVER_KEYWORDS); + } + else{ + throw new Exception("sorry,unknown database tpye..."); + } + } + + public static void transferDatabaseKeywords(List keywords){ + for(int i=0;i columns = context.getColumns(); + // 最后参与排序的索引列 + context.setPkColumns(pkColumns); + int[] pkIndexs = new int[pkColumns.length]; for (int i = 0, n = pkColumns.length; i < n; i++) { String pkc = pkColumns[i]; @@ -131,6 +134,13 @@ public class ObReaderUtils { realIndex.add(columnName); } } + //fix:将主键中的关键字转义 + DatabaseKeywordTransformer.setDatabaseType(DataBaseType.MySql); + if(isOracleMode(context.getCompatibleMode())){ + DatabaseKeywordTransformer.setDatabaseType(DataBaseType.Oracle); + } + DatabaseKeywordTransformer.transferDatabaseKeywords(realIndex); + String[] pks = new String[realIndex.size()]; realIndex.toArray(pks); return pks; From 4e916b0f4b125d1b806206292fa345a984d031ff Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Wed, 22 Dec 2021 17:17:12 +0800 Subject: [PATCH 073/114] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BA=86=E7=94=B1?= =?UTF-8?q?=E4=BA=8E=E6=95=B0=E6=8D=AE=E5=BA=93=E4=BF=9D=E7=95=99=E5=AD=97?= =?UTF-8?q?=E6=98=AF=E8=A1=A8=E5=90=8D=E6=88=96=E8=80=85=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E5=90=8D=E6=97=B6=E5=BC=95=E8=B5=B7=E7=9A=84sql=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/util/DatabaseKeywordTransformer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java index ccbbfb2..2a330cc 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java @@ -27,7 +27,7 @@ public class DatabaseKeywordTransformer { if(databaseType==DataBaseType.MySql){ databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.MYSQL_KEYWORDS); } - else if(databaseType==DataBaseType.Oracle){ + else if(databaseType==DataBaseType.Oracle || databaseType==DataBaseType.OceanBase){ databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.ORACLE_KEYWORDS); } else if(databaseType==DataBaseType.SQLServer){ @@ -45,7 +45,7 @@ public class DatabaseKeywordTransformer { if(databaseType==DataBaseType.MySql){ keyword='`'+keyword+'`'; } - else if(databaseType==DataBaseType.Oracle){ + else if(databaseType==DataBaseType.Oracle || databaseType==DataBaseType.OceanBase){ keyword='"'+keyword+'"'; } else if(databaseType==DataBaseType.SQLServer){ From 738c11e0e3e3e71d57b5b1799e560ecd96d0cb2e Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Wed, 22 Dec 2021 19:54:38 +0800 Subject: [PATCH 074/114] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BA=86=E7=94=B1?= =?UTF-8?q?=E4=BA=8ESQL=E4=B8=AD=E7=9A=84=E4=BF=9D=E7=95=99=E5=AD=97?= =?UTF-8?q?=E4=BD=9C=E4=B8=BA=E8=A1=A8=E5=90=8D=E6=88=96=E8=80=85=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E5=90=8D=E8=80=8C=E5=BC=95=E8=B5=B7=E7=9A=84sql?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/OceanBaseReader.java | 21 +++--- .../util/DatabaseKeywordTransformer.java | 68 ++++++++++--------- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java index db9d34e..04d2848 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java @@ -37,20 +37,17 @@ public class OceanBaseReader extends Reader { if (userConfigedFetchSize != null) { LOG.warn("The [fetchSize] is not recognized, please use readBatchSize instead."); } - this.originalConfig.set(Constant.FETCH_SIZE, Integer.MIN_VALUE); - setDatabaseType(originalConfig); - - - this.readerJob = new ReaderJob(); - this.readerJob.init(this.originalConfig,DATABASE_TYPE); + this.readerJob.init(this.originalConfig, DATABASE_TYPE); } + @Override - public void prepare(){ + public void prepare() { //ObReaderUtils.DATABASE_TYPE获取当前数据库的语法模式 } + @Override public void preCheck() { init(); @@ -80,7 +77,7 @@ public class OceanBaseReader extends Reader { Configuration connConf = Configuration.from(conns.get(0).toString()); List jdbcUrls = connConf.getList(Key.JDBC_URL, String.class); String jdbcUrl = jdbcUrls.get(0); - if(jdbcUrl.startsWith(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING)) { + if (jdbcUrl.startsWith(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING)) { String[] ss = jdbcUrl.split(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING_PATTERN); if (ss.length != 3) { LOG.warn("unrecognized jdbc url: " + jdbcUrl); @@ -97,11 +94,11 @@ public class OceanBaseReader extends Reader { if (ObReaderUtils.isOracleMode(compatibleMode)) { ObReaderUtils.DATABASE_TYPE = DataBaseType.OceanBase; } - } catch (Exception e){ + + } catch (Exception e) { LOG.warn("error in get compatible mode, using mysql as default: " + e.getMessage()); - } - finally { - DATABASE_TYPE=ObReaderUtils.DATABASE_TYPE; + } finally { + DATABASE_TYPE = ObReaderUtils.DATABASE_TYPE; } } } diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java index 2a330cc..6d85cf6 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java @@ -1,65 +1,67 @@ package com.alibaba.datax.plugin.reader.oceanbasev10reader.util; //java api + import java.util.HashSet; import java.util.List; //dataX api import com.alibaba.datax.plugin.rdbms.util.DataBaseType; -//该类用于转义数据库中的关键字 -public class DatabaseKeywordTransformer { +/** + * 该类用于转义数据库中的关键字 + * + * @author:qianzhang + */ +public class DatabaseKeywordTransformer { private static DataBaseType databaseType; static HashSet databaseKeywords; - private static HashSet keywordsFromString2HashSet(final String Keywords){ - String[] keywordArray =Keywords.split(","); - HashSet res=new HashSet(); - for(String keyword: keywordArray){ + + private static HashSet keywordsFromString2HashSet(final String keywords) { + String[] keywordArray = keywords.split(","); + HashSet res = new HashSet(); + for (String keyword : keywordArray) { res.add(keyword); } return res; } + public static void setDatabaseType(final DataBaseType databaseType) throws Exception { - if(databaseType==DatabaseKeywordTransformer.databaseType){ - return ; + if (databaseType == DatabaseKeywordTransformer.databaseType) { + return; } DatabaseKeywordTransformer.databaseType = databaseType; - if(databaseType==DataBaseType.MySql){ - databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.MYSQL_KEYWORDS); - } - else if(databaseType==DataBaseType.Oracle || databaseType==DataBaseType.OceanBase){ - databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.ORACLE_KEYWORDS); - } - else if(databaseType==DataBaseType.SQLServer){ - databaseKeywords=keywordsFromString2HashSet(DatabaseKeywords.SQLSERVER_KEYWORDS); - } - else{ + if (databaseType == DataBaseType.MySql) { + databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.MYSQL_KEYWORDS); + } else if (databaseType == DataBaseType.Oracle || databaseType == DataBaseType.OceanBase) { + databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.ORACLE_KEYWORDS); + } else if (databaseType == DataBaseType.SQLServer) { + databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.SQLSERVER_KEYWORDS); + } else { throw new Exception("sorry,unknown database tpye..."); } } - public static void transferDatabaseKeywords(List keywords){ - for(int i=0;i keywords) { + for (int i = 0; i < keywords.size(); i++) { + String keyword = keywords.get(i).toUpperCase(); + if (databaseKeywords.contains(keyword)) { + if (databaseType == DataBaseType.MySql) { + keyword = '`' + keyword + '`'; + } else if (databaseType == DataBaseType.Oracle || databaseType == DataBaseType.OceanBase) { + keyword = '"' + keyword + '"'; + } else if (databaseType == DataBaseType.SQLServer) { + keyword = '[' + keyword + ']'; } } - keyword=keyword.toLowerCase(); - keywords.set(i,keyword); + keyword = keyword.toLowerCase(); + keywords.set(i, keyword); } } } -final class DatabaseKeywords{ +final class DatabaseKeywords { public static final String MYSQL_KEYWORDS = "ACCESSIBLE,ACCOUNT,ACTION,ADD,AFTER,AGAINST,AGGREGATE,ALGORITHM,ALL,ALTER,ALWAYS,ANALYSE,AND,ANY,AS,ASC,ASCII,ASENSITIVE,AT,AUTO_INCREMENT,AUTOEXTEND_SIZE,AVG,AVG_ROW_LENGTH,BACKUP,BEFORE,BEGIN,BETWEEN,BIGINT,BINARY,BINLOG,BIT,BLOB,BLOCK,BOOL,BOOLEAN,BOTH,BTREE,BY,BYTE,CACHE,CALL,CASCADE,CASCADED,CASE,CATALOG_NAME,CHAIN,CHANGE,CHANGED,CHANNEL,CHAR,CHARACTER,CHARSET,CHECK,CHECKSUM,CIPHER,CLASS_ORIGIN,CLIENT,CLOSE,COALESCE,CODE,COLLATE,COLLATION,COLUMN,COLUMN_FORMAT,COLUMN_NAME,COLUMNS,COMMENT,COMMIT,COMMITTED,COMPACT,COMPLETION,COMPRESSED,COMPRESSION,CONCURRENT,CONDITION,CONNECTION,CONSISTENT,CONSTRAINT,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CONTAINS,CONTEXT,CONTINUE,CONVERT,CPU,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,CURSOR_NAME,DATA,DATABASE,DATABASES,DATAFILE,DATE,DATETIME,DAY,DAY_HOUR,DAY_MICROSECOND,DAY_MINUTE,DAY_SECOND,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_AUTH,DEFINER,DELAY_KEY_WRITE,DELAYED,DELETE,DES_KEY_FILE,DESC,DESCRIBE,DETERMINISTIC,DIAGNOSTICS,DIRECTORY,DISABLE,DISCARD,DISK,DISTINCT,DISTINCTROW,DIV,DO,DOUBLE,DROP,DUAL,DUMPFILE,DUPLICATE,DYNAMIC,EACH,ELSE,ELSEIF,ENABLE,ENCLOSED,ENCRYPTION,END,ENDS,ENGINE,ENGINES,ENUM,ERROR,ERRORS,ESCAPE,ESCAPED,EVENT,EVENTS,EVERY,EXCHANGE,EXECUTE,EXISTS,EXIT,EXPANSION,EXPIRE,EXPLAIN,EXPORT,EXTENDED,EXTENT_SIZE,FAST,FAULTS,FETCH,FIELDS,FILE,FILE_BLOCK_SIZE,FILTER,FIRST,FIXED,FLOAT,FLOAT4,FLOAT8,FLUSH,FOLLOWS,FOR,FORCE,FOREIGN,FORMAT,FOUND,FROM,FULL,FULLTEXT,FUNCTION,GENERAL,GENERATED,GEOMETRY,GEOMETRYCOLLECTION,GET,GET_FORMAT,GLOBAL,GRANT,GRANTS,GROUP,GROUP_REPLICATION,HANDLER,HASH,HAVING,HELP,HIGH_PRIORITY,HOST,HOSTS,HOUR,HOUR_MICROSECOND,HOUR_MINUTE,HOUR_SECOND,IDENTIFIED,IF,IGNORE,IGNORE_SERVER_IDS,IMPORT,IN,INDEX,INDEXES,INFILE,INITIAL_SIZE,INNER,INOUT,INSENSITIVE,INSERT,INSERT_METHOD,INSTALL,INSTANCE,INT,INT1,INT2,INT3,INT4,INT8,INTEGER,INTERVAL,INTO,INVOKER,IO,IO_AFTER_GTIDS,IO_BEFORE_GTIDS,IO_THREAD,IPC,IS,ISOLATION,ISSUER,ITERATE,JOIN,JSON,KEY,KEY_BLOCK_SIZE,KEYS,KILL,LANGUAGE,LAST,LEADING,LEAVE,LEAVES,LEFT,LESS,LEVEL,LIKE,LIMIT,LINEAR,LINES,LINESTRING,LIST,LOAD,LOCAL,LOCALTIME,LOCALTIMESTAMP,LOCK,LOCKS,LOGFILE,LOGS,LONG,LONGBLOB,LONGTEXT,LOOP,LOW_PRIORITY,MASTER,MASTER_AUTO_POSITION,MASTER_BIND,MASTER_CONNECT_RETRY,MASTER_DELAY,MASTER_HEARTBEAT_PERIOD,MASTER_HOST,MASTER_LOG_FILE,MASTER_LOG_POS,MASTER_PASSWORD,MASTER_PORT,MASTER_RETRY_COUNT,MASTER_SERVER_ID,MASTER_SSL,MASTER_SSL_CA,MASTER_SSL_CAPATH,MASTER_SSL_CERT,MASTER_SSL_CIPHER,MASTER_SSL_CRL,MASTER_SSL_CRLPATH,MASTER_SSL_KEY,MASTER_SSL_VERIFY_SERVER_CERT,MASTER_TLS_VERSION,MASTER_USER,MATCH,MAX_CONNECTIONS_PER_HOUR,MAX_QUERIES_PER_HOUR,MAX_ROWS,MAX_SIZE,MAX_STATEMENT_TIME,MAX_UPDATES_PER_HOUR,MAX_USER_CONNECTIONS,MAXVALUE,MEDIUM,MEDIUMBLOB,MEDIUMINT,MEDIUMTEXT,MEMORY,MERGE,MESSAGE_TEXT,MICROSECOND,MIDDLEINT,MIGRATE,MIN_ROWS,MINUTE,MINUTE_MICROSECOND,MINUTE_SECOND,MOD,MODE,MODIFIES,MODIFY,MONTH,MULTILINESTRING,MULTIPOINT,MULTIPOLYGON,MUTEX,MYSQL_ERRNO,NAME,NAMES,NATIONAL,NATURAL,NCHAR,NDB,NDBCLUSTER,NEVER,NEW,NEXT,NO,NO_WAIT,NO_WRITE_TO_BINLOG,NODEGROUP,NONBLOCKING,NONE,NOT,NULL,NUMBER,NUMERIC,NVARCHAR,OFFSET,OLD_PASSWORD,ON,ONE,ONLY,OPEN,OPTIMIZE,OPTIMIZER_COSTS,OPTION,OPTIONALLY,OPTIONS,OR,ORDER,OUT,OUTER,OUTFILE,OWNER,PACK_KEYS,PAGE,PARSE_GCOL_EXPR,PARSER,PARTIAL,PARTITION,PARTITIONING,PARTITIONS,PASSWORD,PHASE,PLUGIN,PLUGIN_DIR,PLUGINS,POINT,POLYGON,PORT,PRECEDES,PRECISION,PREPARE,PRESERVE,PREV,PRIMARY,PRIVILEGES,PROCEDURE,PROCESSLIST,PROFILE,PROFILES,PROXY,PURGE,QUARTER,QUERY,QUICK,RANGE,READ,READ_ONLY,READ_WRITE,READS,REAL,REBUILD,RECOVER,REDO_BUFFER_SIZE,REDOFILE,REDUNDANT,REFERENCES,REGEXP,RELAY,RELAY_LOG_FILE,RELAY_LOG_POS,RELAY_THREAD,RELAYLOG,RELEASE,RELOAD,REMOVE,RENAME,REORGANIZE,REPAIR,REPEAT,REPEATABLE,REPLACE,REPLICATE_DO_DB,REPLICATE_DO_TABLE,REPLICATE_IGNORE_DB,REPLICATE_IGNORE_TABLE,REPLICATE_REWRITE_DB,REPLICATE_WILD_DO_TABLE,REPLICATE_WILD_IGNORE_TABLE,REPLICATION,REQUIRE,RESET,RESIGNAL,RESTORE,RESTRICT,RESUME,RETURN,RETURNED_SQLSTATE,RETURNS,REVERSE,REVOKE,RIGHT,RLIKE,ROLLBACK,ROLLUP,ROTATE,ROUTINE,ROW,ROW_COUNT,ROW_FORMAT,ROWS,RTREE,SAVEPOINT,SCHEDULE,SCHEMA,SCHEMA_NAME,SCHEMAS,SECOND,SECOND_MICROSECOND,SECURITY,SELECT,SENSITIVE,SEPARATOR,SERIAL,SERIALIZABLE,SERVER,SESSION,SET,SHARE,SHOW,SHUTDOWN,SIGNAL,SIGNED,SIMPLE,SLAVE,SLOW,SMALLINT,SNAPSHOT,SOCKET,SOME,SONAME,SOUNDS,SOURCE,SPATIAL,SPECIFIC,SQL,SQL_AFTER_GTIDS,SQL_AFTER_MTS_GAPS,SQL_BEFORE_GTIDS,SQL_BIG_RESULT,SQL_BUFFER_RESULT,SQL_CACHE,SQL_CALC_FOUND_ROWS,SQL_NO_CACHE,SQL_SMALL_RESULT,SQL_THREAD,SQL_TSI_DAY,SQL_TSI_HOUR,SQL_TSI_MINUTE,SQL_TSI_MONTH,SQL_TSI_QUARTER,SQL_TSI_SECOND,SQL_TSI_WEEK,SQL_TSI_YEAR,SQLEXCEPTION,SQLSTATE,SQLWARNING,SSL,STACKED,START,STARTING,STARTS,STATS_AUTO_RECALC,STATS_PERSISTENT,STATS_SAMPLE_PAGES,STATUS,STOP,STORAGE,STORED,STRAIGHT_JOIN,STRING,SUBCLASS_ORIGIN,SUBJECT,SUBPARTITION,SUBPARTITIONS,SUPER,SUSPEND,SWAPS,SWITCHES,TABLE,TABLE_CHECKSUM,TABLE_NAME,TABLES,TABLESPACE,TEMPORARY,TEMPTABLE,TERMINATED,TEXT,THAN,THEN,TIME,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TINYBLOB,TINYINT,TINYTEXT,TO,TRAILING,TRANSACTION,TRIGGER,TRIGGERS,TRUNCATE,TYPE,TYPES,UNCOMMITTED,UNDEFINED,UNDO,UNDO_BUFFER_SIZE,UNDOFILE,UNICODE,UNINSTALL,UNION,UNIQUE,UNKNOWN,UNLOCK,UNSIGNED,UNTIL,UPDATE,UPGRADE,USAGE,USE,USE_FRM,USER,USER_RESOURCES,USING,UTC_DATE,UTC_TIME,UTC_TIMESTAMP,VALIDATION,VALUE,VALUES,VARBINARY,VARCHAR,VARCHARACTER,VARIABLES,VARYING,VIEW,VIRTUAL,WAIT,WARNINGS,WEEK,WEIGHT_STRING,WHEN,WHERE,WHILE,WITH,WITHOUT,WORK,WRAPPER,WRITE,X509,XA,XID,XML,XOR,YEAR,YEAR_MONTH,ZEROFILL,FALSE,TRUE"; From bd43216cf8110703bc7f99ec0d0dcac56ac666fe Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Thu, 23 Dec 2021 13:38:37 +0800 Subject: [PATCH 075/114] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E6=8F=90?= =?UTF-8?q?=E5=88=B0=E7=9A=84=E4=B8=8A=E8=BF=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/ext/ReaderJob.java | 3 +-- .../util/DatabaseKeywordTransformer.java | 19 +++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java index efeec2e..dfa1adb 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java @@ -17,14 +17,13 @@ import org.slf4j.LoggerFactory; public class ReaderJob extends CommonRdbmsReader.Job { - + private Logger LOG=LoggerFactory.getLogger(OceanBaseReader.Task.class); public ReaderJob() { super(ObReaderUtils.DATABASE_TYPE); } public void init(Configuration originalConfig,DataBaseType databaseType){ //将config中的column和table中的关键字进行转义 - final Logger LOG = LoggerFactory.getLogger(OceanBaseReader.Task.class); try { DatabaseKeywordTransformer.setDatabaseType(databaseType); }catch (Exception e){ diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java index 6d85cf6..3c3e03f 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java @@ -2,8 +2,10 @@ package com.alibaba.datax.plugin.reader.oceanbasev10reader.util; //java api +import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; //dataX api import com.alibaba.datax.plugin.rdbms.util.DataBaseType; @@ -15,15 +17,10 @@ import com.alibaba.datax.plugin.rdbms.util.DataBaseType; */ public class DatabaseKeywordTransformer { private static DataBaseType databaseType; - static HashSet databaseKeywords; + static Set databaseKeywords; - private static HashSet keywordsFromString2HashSet(final String keywords) { - String[] keywordArray = keywords.split(","); - HashSet res = new HashSet(); - for (String keyword : keywordArray) { - res.add(keyword); - } - return res; + private static Set keywordsFromString2HashSet(final String keywords) { + return new HashSet(Arrays.asList(keywords.split(","))); } public static void setDatabaseType(final DataBaseType databaseType) throws Exception { @@ -35,8 +32,6 @@ public class DatabaseKeywordTransformer { databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.MYSQL_KEYWORDS); } else if (databaseType == DataBaseType.Oracle || databaseType == DataBaseType.OceanBase) { databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.ORACLE_KEYWORDS); - } else if (databaseType == DataBaseType.SQLServer) { - databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.SQLSERVER_KEYWORDS); } else { throw new Exception("sorry,unknown database tpye..."); } @@ -50,8 +45,6 @@ public class DatabaseKeywordTransformer { keyword = '`' + keyword + '`'; } else if (databaseType == DataBaseType.Oracle || databaseType == DataBaseType.OceanBase) { keyword = '"' + keyword + '"'; - } else if (databaseType == DataBaseType.SQLServer) { - keyword = '[' + keyword + ']'; } } keyword = keyword.toLowerCase(); @@ -65,7 +58,5 @@ final class DatabaseKeywords { public static final String MYSQL_KEYWORDS = "ACCESSIBLE,ACCOUNT,ACTION,ADD,AFTER,AGAINST,AGGREGATE,ALGORITHM,ALL,ALTER,ALWAYS,ANALYSE,AND,ANY,AS,ASC,ASCII,ASENSITIVE,AT,AUTO_INCREMENT,AUTOEXTEND_SIZE,AVG,AVG_ROW_LENGTH,BACKUP,BEFORE,BEGIN,BETWEEN,BIGINT,BINARY,BINLOG,BIT,BLOB,BLOCK,BOOL,BOOLEAN,BOTH,BTREE,BY,BYTE,CACHE,CALL,CASCADE,CASCADED,CASE,CATALOG_NAME,CHAIN,CHANGE,CHANGED,CHANNEL,CHAR,CHARACTER,CHARSET,CHECK,CHECKSUM,CIPHER,CLASS_ORIGIN,CLIENT,CLOSE,COALESCE,CODE,COLLATE,COLLATION,COLUMN,COLUMN_FORMAT,COLUMN_NAME,COLUMNS,COMMENT,COMMIT,COMMITTED,COMPACT,COMPLETION,COMPRESSED,COMPRESSION,CONCURRENT,CONDITION,CONNECTION,CONSISTENT,CONSTRAINT,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CONTAINS,CONTEXT,CONTINUE,CONVERT,CPU,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,CURSOR_NAME,DATA,DATABASE,DATABASES,DATAFILE,DATE,DATETIME,DAY,DAY_HOUR,DAY_MICROSECOND,DAY_MINUTE,DAY_SECOND,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_AUTH,DEFINER,DELAY_KEY_WRITE,DELAYED,DELETE,DES_KEY_FILE,DESC,DESCRIBE,DETERMINISTIC,DIAGNOSTICS,DIRECTORY,DISABLE,DISCARD,DISK,DISTINCT,DISTINCTROW,DIV,DO,DOUBLE,DROP,DUAL,DUMPFILE,DUPLICATE,DYNAMIC,EACH,ELSE,ELSEIF,ENABLE,ENCLOSED,ENCRYPTION,END,ENDS,ENGINE,ENGINES,ENUM,ERROR,ERRORS,ESCAPE,ESCAPED,EVENT,EVENTS,EVERY,EXCHANGE,EXECUTE,EXISTS,EXIT,EXPANSION,EXPIRE,EXPLAIN,EXPORT,EXTENDED,EXTENT_SIZE,FAST,FAULTS,FETCH,FIELDS,FILE,FILE_BLOCK_SIZE,FILTER,FIRST,FIXED,FLOAT,FLOAT4,FLOAT8,FLUSH,FOLLOWS,FOR,FORCE,FOREIGN,FORMAT,FOUND,FROM,FULL,FULLTEXT,FUNCTION,GENERAL,GENERATED,GEOMETRY,GEOMETRYCOLLECTION,GET,GET_FORMAT,GLOBAL,GRANT,GRANTS,GROUP,GROUP_REPLICATION,HANDLER,HASH,HAVING,HELP,HIGH_PRIORITY,HOST,HOSTS,HOUR,HOUR_MICROSECOND,HOUR_MINUTE,HOUR_SECOND,IDENTIFIED,IF,IGNORE,IGNORE_SERVER_IDS,IMPORT,IN,INDEX,INDEXES,INFILE,INITIAL_SIZE,INNER,INOUT,INSENSITIVE,INSERT,INSERT_METHOD,INSTALL,INSTANCE,INT,INT1,INT2,INT3,INT4,INT8,INTEGER,INTERVAL,INTO,INVOKER,IO,IO_AFTER_GTIDS,IO_BEFORE_GTIDS,IO_THREAD,IPC,IS,ISOLATION,ISSUER,ITERATE,JOIN,JSON,KEY,KEY_BLOCK_SIZE,KEYS,KILL,LANGUAGE,LAST,LEADING,LEAVE,LEAVES,LEFT,LESS,LEVEL,LIKE,LIMIT,LINEAR,LINES,LINESTRING,LIST,LOAD,LOCAL,LOCALTIME,LOCALTIMESTAMP,LOCK,LOCKS,LOGFILE,LOGS,LONG,LONGBLOB,LONGTEXT,LOOP,LOW_PRIORITY,MASTER,MASTER_AUTO_POSITION,MASTER_BIND,MASTER_CONNECT_RETRY,MASTER_DELAY,MASTER_HEARTBEAT_PERIOD,MASTER_HOST,MASTER_LOG_FILE,MASTER_LOG_POS,MASTER_PASSWORD,MASTER_PORT,MASTER_RETRY_COUNT,MASTER_SERVER_ID,MASTER_SSL,MASTER_SSL_CA,MASTER_SSL_CAPATH,MASTER_SSL_CERT,MASTER_SSL_CIPHER,MASTER_SSL_CRL,MASTER_SSL_CRLPATH,MASTER_SSL_KEY,MASTER_SSL_VERIFY_SERVER_CERT,MASTER_TLS_VERSION,MASTER_USER,MATCH,MAX_CONNECTIONS_PER_HOUR,MAX_QUERIES_PER_HOUR,MAX_ROWS,MAX_SIZE,MAX_STATEMENT_TIME,MAX_UPDATES_PER_HOUR,MAX_USER_CONNECTIONS,MAXVALUE,MEDIUM,MEDIUMBLOB,MEDIUMINT,MEDIUMTEXT,MEMORY,MERGE,MESSAGE_TEXT,MICROSECOND,MIDDLEINT,MIGRATE,MIN_ROWS,MINUTE,MINUTE_MICROSECOND,MINUTE_SECOND,MOD,MODE,MODIFIES,MODIFY,MONTH,MULTILINESTRING,MULTIPOINT,MULTIPOLYGON,MUTEX,MYSQL_ERRNO,NAME,NAMES,NATIONAL,NATURAL,NCHAR,NDB,NDBCLUSTER,NEVER,NEW,NEXT,NO,NO_WAIT,NO_WRITE_TO_BINLOG,NODEGROUP,NONBLOCKING,NONE,NOT,NULL,NUMBER,NUMERIC,NVARCHAR,OFFSET,OLD_PASSWORD,ON,ONE,ONLY,OPEN,OPTIMIZE,OPTIMIZER_COSTS,OPTION,OPTIONALLY,OPTIONS,OR,ORDER,OUT,OUTER,OUTFILE,OWNER,PACK_KEYS,PAGE,PARSE_GCOL_EXPR,PARSER,PARTIAL,PARTITION,PARTITIONING,PARTITIONS,PASSWORD,PHASE,PLUGIN,PLUGIN_DIR,PLUGINS,POINT,POLYGON,PORT,PRECEDES,PRECISION,PREPARE,PRESERVE,PREV,PRIMARY,PRIVILEGES,PROCEDURE,PROCESSLIST,PROFILE,PROFILES,PROXY,PURGE,QUARTER,QUERY,QUICK,RANGE,READ,READ_ONLY,READ_WRITE,READS,REAL,REBUILD,RECOVER,REDO_BUFFER_SIZE,REDOFILE,REDUNDANT,REFERENCES,REGEXP,RELAY,RELAY_LOG_FILE,RELAY_LOG_POS,RELAY_THREAD,RELAYLOG,RELEASE,RELOAD,REMOVE,RENAME,REORGANIZE,REPAIR,REPEAT,REPEATABLE,REPLACE,REPLICATE_DO_DB,REPLICATE_DO_TABLE,REPLICATE_IGNORE_DB,REPLICATE_IGNORE_TABLE,REPLICATE_REWRITE_DB,REPLICATE_WILD_DO_TABLE,REPLICATE_WILD_IGNORE_TABLE,REPLICATION,REQUIRE,RESET,RESIGNAL,RESTORE,RESTRICT,RESUME,RETURN,RETURNED_SQLSTATE,RETURNS,REVERSE,REVOKE,RIGHT,RLIKE,ROLLBACK,ROLLUP,ROTATE,ROUTINE,ROW,ROW_COUNT,ROW_FORMAT,ROWS,RTREE,SAVEPOINT,SCHEDULE,SCHEMA,SCHEMA_NAME,SCHEMAS,SECOND,SECOND_MICROSECOND,SECURITY,SELECT,SENSITIVE,SEPARATOR,SERIAL,SERIALIZABLE,SERVER,SESSION,SET,SHARE,SHOW,SHUTDOWN,SIGNAL,SIGNED,SIMPLE,SLAVE,SLOW,SMALLINT,SNAPSHOT,SOCKET,SOME,SONAME,SOUNDS,SOURCE,SPATIAL,SPECIFIC,SQL,SQL_AFTER_GTIDS,SQL_AFTER_MTS_GAPS,SQL_BEFORE_GTIDS,SQL_BIG_RESULT,SQL_BUFFER_RESULT,SQL_CACHE,SQL_CALC_FOUND_ROWS,SQL_NO_CACHE,SQL_SMALL_RESULT,SQL_THREAD,SQL_TSI_DAY,SQL_TSI_HOUR,SQL_TSI_MINUTE,SQL_TSI_MONTH,SQL_TSI_QUARTER,SQL_TSI_SECOND,SQL_TSI_WEEK,SQL_TSI_YEAR,SQLEXCEPTION,SQLSTATE,SQLWARNING,SSL,STACKED,START,STARTING,STARTS,STATS_AUTO_RECALC,STATS_PERSISTENT,STATS_SAMPLE_PAGES,STATUS,STOP,STORAGE,STORED,STRAIGHT_JOIN,STRING,SUBCLASS_ORIGIN,SUBJECT,SUBPARTITION,SUBPARTITIONS,SUPER,SUSPEND,SWAPS,SWITCHES,TABLE,TABLE_CHECKSUM,TABLE_NAME,TABLES,TABLESPACE,TEMPORARY,TEMPTABLE,TERMINATED,TEXT,THAN,THEN,TIME,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TINYBLOB,TINYINT,TINYTEXT,TO,TRAILING,TRANSACTION,TRIGGER,TRIGGERS,TRUNCATE,TYPE,TYPES,UNCOMMITTED,UNDEFINED,UNDO,UNDO_BUFFER_SIZE,UNDOFILE,UNICODE,UNINSTALL,UNION,UNIQUE,UNKNOWN,UNLOCK,UNSIGNED,UNTIL,UPDATE,UPGRADE,USAGE,USE,USE_FRM,USER,USER_RESOURCES,USING,UTC_DATE,UTC_TIME,UTC_TIMESTAMP,VALIDATION,VALUE,VALUES,VARBINARY,VARCHAR,VARCHARACTER,VARIABLES,VARYING,VIEW,VIRTUAL,WAIT,WARNINGS,WEEK,WEIGHT_STRING,WHEN,WHERE,WHILE,WITH,WITHOUT,WORK,WRAPPER,WRITE,X509,XA,XID,XML,XOR,YEAR,YEAR_MONTH,ZEROFILL,FALSE,TRUE"; - public static final String SQLSERVER_KEYWORDS = "ADD,ALL,ALTER,AND,ANY,AS,ASC,AUTHORIZATION,BACKUP,BEGIN,BETWEEN,BREAK,BROWSE,BULK,BY,CASCADE,CASE,CHECK,CHECKPOINT,CLOSE,CLUSTERED,COALESCE,COLLATE,COLUMN,COMMIT,COMPUTE,CONSTRAINT,CONTAINS,CONTAINSTABLE,CONTINUE,CONVERT,CREATE,CROSS,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATABASE,DBCC,DEALLOCATE,DECLARE,DEFAULT,DELETE,DENY,DESC,DISK,DISTINCT,DISTRIBUTED,DOUBLE,DROP,DUMMY,DUMP,ELSE,END,ERRLVL,ESCAPE,EXCEPT,EXEC,EXECUTE,EXISTS,EXIT,FETCH,FILE,FILLFACTOR,FOR,FOREIGN,FREETEXT,FREETEXTTABLE,FROM,FULL,FUNCTION,GOTO,GRANT,GROUP,HAVING,HOLDLOCK,IDENTITY,IDENTITY_INSERT,IDENTITYCOL,IF,IN,INDEX,INNER,INSERT,INTERSECT,INTO,IS,JOIN,KEY,KILL,LEFT,LIKE,LINENO,LOAD,NATIONAL,NOCHECK,NONCLUSTERED,NOT,NULL,NULLIF,OF,OFF,OFFSETS,ON,OPEN,OPENDATASOURCE,OPENQUERY,OPENROWSET,OPENXML,OPTION,OR,ORDER,OUTER,OVER,PERCENT,PLAN,PRECISION,PRIMARY,PRINT,PROC,PROCEDURE,PUBLIC,RAISERROR,READ,READTEXT,RECONFIGURE,REFERENCES,REPLICATION,RESTORE,RESTRICT,RETURN,REVOKE,RIGHT,ROLLBACK,ROWCOUNT,ROWGUIDCOL,RULE,SAVE,SCHEMA,SELECT,SESSION_USER,SET,SETUSER,SHUTDOWN,SOME,STATISTICS,SYSTEM_USER,TABLE,TEXTSIZE,THEN,TO,TOP,TRAN,TRANSACTION,TRIGGER,TRUNCATE,TSEQUAL,UNION,UNIQUE,UPDATE,UPDATETEXT,USE,USER,VALUES,VARYING,VIEW,WAITFOR,WHEN,WHERE,WHILE,WITH,WRITETEXT"; - public static final String ORACLE_KEYWORDS = "ACCESS,ADD,ALL,ALTER,AND,ANY,ARRAYLEN,AS,ASC,AUDIT,BETWEEN,BY,CHAR,CHECK,CLUSTER,COLUMN,COMMENT,COMPRESS,CONNECT,CREATE,CURRENT,DATE,DECIMAL,DEFAULT,DELETE,DESC,DISTINCT,DROP,ELSE,EXCLUSIVE,EXISTS,FILE,FLOAT,FOR,FROM,GRANT,GROUP,HAVING,IDENTIFIED,IMMEDIATE,IN,INCREMENT,INDEX,INITIAL,INSERT,INTEGER,INTERSECT,INTO,IS,LEVEL,LIKE,LOCK,LONG,MAXEXTENTS,MINUS,MODE,MODIFY,NOAUDIT,NOCOMPRESS,NOT,NOTFOUND,NOWAIT,NULL,NUMBER,OF,OFFLINE,ON,ONLINE,OPTION,OR,ORDER,PCTFREE,PRIOR,PRIVILEGES,PUBLIC,RAW,RENAME,RESOURCE,REVOKE,ROW,ROWID,ROWLABEL,ROWNUM,ROWS,SELECT,SESSION,SET,SHARE,SIZE,SMALLINT,SQLBUF,START,SUCCESSFUL,SYNONYM,TABLE,THEN,TO,TRIGGER,UID,UNION,UNIQUE,UPDATE,USER,VALIDATE,VALUES,VARCHAR,VARCHAR2,VIEW,WHENEVER,WHERE,WITH"; } \ No newline at end of file From 30ed711131c84f8401e06e186434f5ed340f647f Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Thu, 23 Dec 2021 13:46:26 +0800 Subject: [PATCH 076/114] =?UTF-8?q?=E5=86=8D=E6=AC=A1=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BA=86=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java index dfa1adb..2584ca3 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java @@ -26,8 +26,8 @@ public class ReaderJob extends CommonRdbmsReader.Job { //将config中的column和table中的关键字进行转义 try { DatabaseKeywordTransformer.setDatabaseType(databaseType); - }catch (Exception e){ - LOG.warn("database type is "+databaseType+e.getMessage()); + } catch (Exception e) { + LOG.warn("database type is " + databaseType + e.getMessage()); } List columns=originalConfig.getList(Key.COLUMN,String.class); DatabaseKeywordTransformer.transferDatabaseKeywords(columns); From b017102e0e59ec36e9863a2a2fa90d55a376d5ad Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Thu, 23 Dec 2021 15:20:56 +0800 Subject: [PATCH 077/114] =?UTF-8?q?=E6=9C=80=E7=BB=88=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/ext/ReaderJob.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java index 2584ca3..d574e3e 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java @@ -17,32 +17,35 @@ import org.slf4j.LoggerFactory; public class ReaderJob extends CommonRdbmsReader.Job { - private Logger LOG=LoggerFactory.getLogger(OceanBaseReader.Task.class); + private Logger LOG = LoggerFactory.getLogger(OceanBaseReader.Task.class); + public ReaderJob() { super(ObReaderUtils.DATABASE_TYPE); } - public void init(Configuration originalConfig,DataBaseType databaseType){ + + public void init(Configuration originalConfig, DataBaseType databaseType) { //将config中的column和table中的关键字进行转义 try { DatabaseKeywordTransformer.setDatabaseType(databaseType); } catch (Exception e) { LOG.warn("database type is " + databaseType + e.getMessage()); } - List columns=originalConfig.getList(Key.COLUMN,String.class); + List columns = originalConfig.getList(Key.COLUMN, String.class); DatabaseKeywordTransformer.transferDatabaseKeywords(columns); originalConfig.set(Key.COLUMN, columns); - List conns=originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK,JSONObject.class); - for(int i=0;i tables=connConfig.getList(Key.TABLE,String.class); + List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, JSONObject.class); + for (int i = 0; i < conns.size(); i++) { + JSONObject conn = conns.get(i); + Configuration connConfig = Configuration.from(conn.toString()); + List tables = connConfig.getList(Key.TABLE, String.class); DatabaseKeywordTransformer.transferDatabaseKeywords(tables); - originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK,i,Key.TABLE),tables); + originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, i, Key.TABLE), tables); } super.init(originalConfig); } + @Override public List split(Configuration originalConfig, int adviceNumber) { List list = super.split(originalConfig, adviceNumber); From 19a44e8d9d1f40e1fbf4b2b0b105f346c00fbfbe Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Thu, 23 Dec 2021 18:10:08 +0800 Subject: [PATCH 078/114] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=86=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E7=9A=84=E4=B8=80=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/OceanBaseReader.java | 14 +- .../oceanbasev10reader/ext/ReaderJob.java | 16 +-- .../oceanbasev10reader/ext/ReaderTask.java | 24 ++-- .../util/ObReaderUtils.java | 125 ++++++++++-------- .../oceanbasev10reader/util/TaskContext.java | 1 + 5 files changed, 94 insertions(+), 86 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java index 04d2848..fc46159 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java @@ -1,12 +1,8 @@ package com.alibaba.datax.plugin.reader.oceanbasev10reader; -import java.sql.Array; import java.sql.Connection; -import java.util.ArrayList; import java.util.List; -import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.DatabaseKeywordTransformer; -import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +22,6 @@ public class OceanBaseReader extends Reader { public static class Job extends Reader.Job { private Configuration originalConfig = null; private ReaderJob readerJob; - private DataBaseType DATABASE_TYPE; private static final Logger LOG = LoggerFactory.getLogger(Task.class); @Override @@ -40,7 +35,7 @@ public class OceanBaseReader extends Reader { this.originalConfig.set(Constant.FETCH_SIZE, Integer.MIN_VALUE); setDatabaseType(originalConfig); this.readerJob = new ReaderJob(); - this.readerJob.init(this.originalConfig, DATABASE_TYPE); + this.readerJob.init(this.originalConfig); } @Override @@ -51,7 +46,7 @@ public class OceanBaseReader extends Reader { @Override public void preCheck() { init(); - this.readerJob.preCheck(this.originalConfig, ObReaderUtils.DATABASE_TYPE); + this.readerJob.preCheck(this.originalConfig, ObReaderUtils.compatibleMode); } @@ -92,13 +87,12 @@ public class OceanBaseReader extends Reader { Connection conn = DBUtil.getConnection(DataBaseType.OceanBase, obJdbcUrl, username, password); String compatibleMode = ObReaderUtils.getCompatibleMode(conn); if (ObReaderUtils.isOracleMode(compatibleMode)) { - ObReaderUtils.DATABASE_TYPE = DataBaseType.OceanBase; + ObReaderUtils.compatibleMode = DataBaseType.Oracle; + ObReaderUtils.databaseType = DataBaseType.OceanBase; } } catch (Exception e) { LOG.warn("error in get compatible mode, using mysql as default: " + e.getMessage()); - } finally { - DATABASE_TYPE = ObReaderUtils.DATABASE_TYPE; } } } diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java index d574e3e..2a31aaa 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java @@ -9,7 +9,6 @@ import com.alibaba.datax.plugin.rdbms.reader.Key; import com.alibaba.datax.plugin.rdbms.util.DataBaseType; import com.alibaba.datax.plugin.rdbms.writer.Constant; import com.alibaba.datax.plugin.reader.oceanbasev10reader.OceanBaseReader; -import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.DatabaseKeywordTransformer; import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.ObReaderUtils; import com.alibaba.fastjson.JSONObject; import org.slf4j.Logger; @@ -20,19 +19,14 @@ public class ReaderJob extends CommonRdbmsReader.Job { private Logger LOG = LoggerFactory.getLogger(OceanBaseReader.Task.class); public ReaderJob() { - super(ObReaderUtils.DATABASE_TYPE); - + super(ObReaderUtils.databaseType); } - public void init(Configuration originalConfig, DataBaseType databaseType) { + @Override + public void init(Configuration originalConfig) { //将config中的column和table中的关键字进行转义 - try { - DatabaseKeywordTransformer.setDatabaseType(databaseType); - } catch (Exception e) { - LOG.warn("database type is " + databaseType + e.getMessage()); - } List columns = originalConfig.getList(Key.COLUMN, String.class); - DatabaseKeywordTransformer.transferDatabaseKeywords(columns); + ObReaderUtils.transferDatabaseKeywords(columns); originalConfig.set(Key.COLUMN, columns); List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, JSONObject.class); @@ -40,7 +34,7 @@ public class ReaderJob extends CommonRdbmsReader.Job { JSONObject conn = conns.get(i); Configuration connConfig = Configuration.from(conn.toString()); List tables = connConfig.getList(Key.TABLE, String.class); - DatabaseKeywordTransformer.transferDatabaseKeywords(tables); + ObReaderUtils.transferDatabaseKeywords(tables); originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, i, Key.TABLE), tables); } super.init(originalConfig); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java index 073bb3c..b5314ad 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java @@ -41,11 +41,12 @@ public class ReaderTask extends CommonRdbmsReader.Task { private boolean reuseConn = false; public ReaderTask(int taskGroupId, int taskId) { - super(ObReaderUtils.DATABASE_TYPE, taskGroupId, taskId); + super(ObReaderUtils.compatibleMode, taskGroupId, taskId); this.taskGroupId = taskGroupId; this.taskId = taskId; } + @Override public void init(Configuration readerSliceConfig) { /* for database connection */ username = readerSliceConfig.getString(Key.USERNAME); @@ -54,7 +55,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { queryTimeoutSeconds = readerSliceConfig.getInt(Config.QUERY_TIMEOUT_SECOND, Config.DEFAULT_QUERY_TIMEOUT_SECOND); // ob10的处理 - if(jdbcUrl.startsWith(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING)) { + if (jdbcUrl.startsWith(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING)) { String[] ss = jdbcUrl.split(com.alibaba.datax.plugin.rdbms.writer.Constant.OB10_SPLIT_STRING_PATTERN); if (ss.length == 3) { LOG.info("this is ob1_0 jdbc url."); @@ -63,7 +64,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { } } - if (ObReaderUtils.DATABASE_TYPE == DataBaseType.OceanBase) { + if (ObReaderUtils.databaseType == DataBaseType.OceanBase) { jdbcUrl = jdbcUrl.replace("jdbc:mysql:", "jdbc:oceanbase:") + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 compatibleMode = ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; } else { @@ -72,7 +73,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { LOG.info("this is ob1_0 jdbc url. user=" + username + " :url=" + jdbcUrl); mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); retryLimit = readerSliceConfig.getInt(Config.RETRY_LIMIT, Config.DEFAULT_RETRY_LIMIT); - LOG.info("retryLimit: "+ retryLimit); + LOG.info("retryLimit: " + retryLimit); } private void buildSavePoint(TaskContext context) { @@ -83,7 +84,6 @@ public class ReaderTask extends CommonRdbmsReader.Task { } /** - * * 如果isTableMode && table有PK *

* 则支持断点续读 (若pk不在原始的columns中,则追加到尾部,但不传给下游) @@ -92,7 +92,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { */ @Override public void startRead(Configuration readerSliceConfig, RecordSender recordSender, - TaskPluginCollector taskPluginCollector, int fetchSize) { + TaskPluginCollector taskPluginCollector, int fetchSize) { String querySql = readerSliceConfig.getString(Key.QUERY_SQL); String table = readerSliceConfig.getString(Key.TABLE); PerfTrace.getInstance().addTaskDetails(taskId, table + "," + jdbcUrl); @@ -131,14 +131,14 @@ public class ReaderTask extends CommonRdbmsReader.Task { } private void startRead0(boolean isTableMode, TaskContext context, RecordSender recordSender, - TaskPluginCollector taskPluginCollector) { + TaskPluginCollector taskPluginCollector) { // 不是table模式 直接使用原来的做法 if (!isTableMode) { doRead(recordSender, taskPluginCollector, context); return; } // check primary key index - Connection conn = DBUtil.getConnection(ObReaderUtils.DATABASE_TYPE, jdbcUrl, username, password); + Connection conn = DBUtil.getConnection(ObReaderUtils.databaseType, jdbcUrl, username, password); ObReaderUtils.initConn4Reader(conn, queryTimeoutSeconds); context.setConn(conn); try { @@ -184,11 +184,11 @@ public class ReaderTask extends CommonRdbmsReader.Task { } } catch (Throwable e) { if (retryLimit == ++retryCount) { - throw RdbmsException.asQueryException(ObReaderUtils.DATABASE_TYPE, new Exception(e), + throw RdbmsException.asQueryException(ObReaderUtils.compatibleMode, new Exception(e), context.getQuerySql(), context.getTable(), username); } LOG.error("read fail, retry count " + retryCount + ", sleep 60 second, save point:" + - context.getSavePoint() + ", error: "+ e.getMessage()); + context.getSavePoint() + ", error: " + e.getMessage()); ObReaderUtils.sleep(60000); // sleep 10s } // 假如原来的查询有查出数据,则改成增量查询 @@ -227,7 +227,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { LOG.info("connection is alive, will reuse this connection."); } else { LOG.info("Create new connection for reader."); - conn = DBUtil.getConnection(ObReaderUtils.DATABASE_TYPE, jdbcUrl, username, password); + conn = DBUtil.getConnection(ObReaderUtils.databaseType, jdbcUrl, username, password); ObReaderUtils.initConn4Reader(conn, queryTimeoutSeconds); context.setConn(conn); } @@ -287,7 +287,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { ObReaderUtils.close(null, null, context.getConn()); context.setConn(null); LOG.error("reader data fail", e); - throw RdbmsException.asQueryException(ObReaderUtils.DATABASE_TYPE, e, context.getQuerySql(), + throw RdbmsException.asQueryException(ObReaderUtils.compatibleMode, e, context.getQuerySql(), context.getTable(), username); } finally { perfRecord.end(); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java index 39f12ea..5084523 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java @@ -6,18 +6,12 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.Set; -import java.util.TreeSet; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -32,21 +26,54 @@ import com.alibaba.datax.common.element.LongColumn; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.element.StringColumn; import com.alibaba.datax.plugin.rdbms.util.DBUtil; -import com.alibaba.datax.plugin.rdbms.util.DataBaseType; import com.alibaba.druid.sql.SQLUtils; import com.alibaba.druid.sql.ast.SQLExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; +import javax.xml.crypto.Data; + public class ObReaderUtils { private static final Logger LOG = LoggerFactory.getLogger(ObReaderUtils.class); + private static final String MYSQL_KEYWORDS = "ACCESSIBLE,ACCOUNT,ACTION,ADD,AFTER,AGAINST,AGGREGATE,ALGORITHM,ALL,ALTER,ALWAYS,ANALYSE,AND,ANY,AS,ASC,ASCII,ASENSITIVE,AT,AUTO_INCREMENT,AUTOEXTEND_SIZE,AVG,AVG_ROW_LENGTH,BACKUP,BEFORE,BEGIN,BETWEEN,BIGINT,BINARY,BINLOG,BIT,BLOB,BLOCK,BOOL,BOOLEAN,BOTH,BTREE,BY,BYTE,CACHE,CALL,CASCADE,CASCADED,CASE,CATALOG_NAME,CHAIN,CHANGE,CHANGED,CHANNEL,CHAR,CHARACTER,CHARSET,CHECK,CHECKSUM,CIPHER,CLASS_ORIGIN,CLIENT,CLOSE,COALESCE,CODE,COLLATE,COLLATION,COLUMN,COLUMN_FORMAT,COLUMN_NAME,COLUMNS,COMMENT,COMMIT,COMMITTED,COMPACT,COMPLETION,COMPRESSED,COMPRESSION,CONCURRENT,CONDITION,CONNECTION,CONSISTENT,CONSTRAINT,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CONTAINS,CONTEXT,CONTINUE,CONVERT,CPU,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,CURSOR_NAME,DATA,DATABASE,DATABASES,DATAFILE,DATE,DATETIME,DAY,DAY_HOUR,DAY_MICROSECOND,DAY_MINUTE,DAY_SECOND,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_AUTH,DEFINER,DELAY_KEY_WRITE,DELAYED,DELETE,DES_KEY_FILE,DESC,DESCRIBE,DETERMINISTIC,DIAGNOSTICS,DIRECTORY,DISABLE,DISCARD,DISK,DISTINCT,DISTINCTROW,DIV,DO,DOUBLE,DROP,DUAL,DUMPFILE,DUPLICATE,DYNAMIC,EACH,ELSE,ELSEIF,ENABLE,ENCLOSED,ENCRYPTION,END,ENDS,ENGINE,ENGINES,ENUM,ERROR,ERRORS,ESCAPE,ESCAPED,EVENT,EVENTS,EVERY,EXCHANGE,EXECUTE,EXISTS,EXIT,EXPANSION,EXPIRE,EXPLAIN,EXPORT,EXTENDED,EXTENT_SIZE,FAST,FAULTS,FETCH,FIELDS,FILE,FILE_BLOCK_SIZE,FILTER,FIRST,FIXED,FLOAT,FLOAT4,FLOAT8,FLUSH,FOLLOWS,FOR,FORCE,FOREIGN,FORMAT,FOUND,FROM,FULL,FULLTEXT,FUNCTION,GENERAL,GENERATED,GEOMETRY,GEOMETRYCOLLECTION,GET,GET_FORMAT,GLOBAL,GRANT,GRANTS,GROUP,GROUP_REPLICATION,HANDLER,HASH,HAVING,HELP,HIGH_PRIORITY,HOST,HOSTS,HOUR,HOUR_MICROSECOND,HOUR_MINUTE,HOUR_SECOND,IDENTIFIED,IF,IGNORE,IGNORE_SERVER_IDS,IMPORT,IN,INDEX,INDEXES,INFILE,INITIAL_SIZE,INNER,INOUT,INSENSITIVE,INSERT,INSERT_METHOD,INSTALL,INSTANCE,INT,INT1,INT2,INT3,INT4,INT8,INTEGER,INTERVAL,INTO,INVOKER,IO,IO_AFTER_GTIDS,IO_BEFORE_GTIDS,IO_THREAD,IPC,IS,ISOLATION,ISSUER,ITERATE,JOIN,JSON,KEY,KEY_BLOCK_SIZE,KEYS,KILL,LANGUAGE,LAST,LEADING,LEAVE,LEAVES,LEFT,LESS,LEVEL,LIKE,LIMIT,LINEAR,LINES,LINESTRING,LIST,LOAD,LOCAL,LOCALTIME,LOCALTIMESTAMP,LOCK,LOCKS,LOGFILE,LOGS,LONG,LONGBLOB,LONGTEXT,LOOP,LOW_PRIORITY,MASTER,MASTER_AUTO_POSITION,MASTER_BIND,MASTER_CONNECT_RETRY,MASTER_DELAY,MASTER_HEARTBEAT_PERIOD,MASTER_HOST,MASTER_LOG_FILE,MASTER_LOG_POS,MASTER_PASSWORD,MASTER_PORT,MASTER_RETRY_COUNT,MASTER_SERVER_ID,MASTER_SSL,MASTER_SSL_CA,MASTER_SSL_CAPATH,MASTER_SSL_CERT,MASTER_SSL_CIPHER,MASTER_SSL_CRL,MASTER_SSL_CRLPATH,MASTER_SSL_KEY,MASTER_SSL_VERIFY_SERVER_CERT,MASTER_TLS_VERSION,MASTER_USER,MATCH,MAX_CONNECTIONS_PER_HOUR,MAX_QUERIES_PER_HOUR,MAX_ROWS,MAX_SIZE,MAX_STATEMENT_TIME,MAX_UPDATES_PER_HOUR,MAX_USER_CONNECTIONS,MAXVALUE,MEDIUM,MEDIUMBLOB,MEDIUMINT,MEDIUMTEXT,MEMORY,MERGE,MESSAGE_TEXT,MICROSECOND,MIDDLEINT,MIGRATE,MIN_ROWS,MINUTE,MINUTE_MICROSECOND,MINUTE_SECOND,MOD,MODE,MODIFIES,MODIFY,MONTH,MULTILINESTRING,MULTIPOINT,MULTIPOLYGON,MUTEX,MYSQL_ERRNO,NAME,NAMES,NATIONAL,NATURAL,NCHAR,NDB,NDBCLUSTER,NEVER,NEW,NEXT,NO,NO_WAIT,NO_WRITE_TO_BINLOG,NODEGROUP,NONBLOCKING,NONE,NOT,NULL,NUMBER,NUMERIC,NVARCHAR,OFFSET,OLD_PASSWORD,ON,ONE,ONLY,OPEN,OPTIMIZE,OPTIMIZER_COSTS,OPTION,OPTIONALLY,OPTIONS,OR,ORDER,OUT,OUTER,OUTFILE,OWNER,PACK_KEYS,PAGE,PARSE_GCOL_EXPR,PARSER,PARTIAL,PARTITION,PARTITIONING,PARTITIONS,PASSWORD,PHASE,PLUGIN,PLUGIN_DIR,PLUGINS,POINT,POLYGON,PORT,PRECEDES,PRECISION,PREPARE,PRESERVE,PREV,PRIMARY,PRIVILEGES,PROCEDURE,PROCESSLIST,PROFILE,PROFILES,PROXY,PURGE,QUARTER,QUERY,QUICK,RANGE,READ,READ_ONLY,READ_WRITE,READS,REAL,REBUILD,RECOVER,REDO_BUFFER_SIZE,REDOFILE,REDUNDANT,REFERENCES,REGEXP,RELAY,RELAY_LOG_FILE,RELAY_LOG_POS,RELAY_THREAD,RELAYLOG,RELEASE,RELOAD,REMOVE,RENAME,REORGANIZE,REPAIR,REPEAT,REPEATABLE,REPLACE,REPLICATE_DO_DB,REPLICATE_DO_TABLE,REPLICATE_IGNORE_DB,REPLICATE_IGNORE_TABLE,REPLICATE_REWRITE_DB,REPLICATE_WILD_DO_TABLE,REPLICATE_WILD_IGNORE_TABLE,REPLICATION,REQUIRE,RESET,RESIGNAL,RESTORE,RESTRICT,RESUME,RETURN,RETURNED_SQLSTATE,RETURNS,REVERSE,REVOKE,RIGHT,RLIKE,ROLLBACK,ROLLUP,ROTATE,ROUTINE,ROW,ROW_COUNT,ROW_FORMAT,ROWS,RTREE,SAVEPOINT,SCHEDULE,SCHEMA,SCHEMA_NAME,SCHEMAS,SECOND,SECOND_MICROSECOND,SECURITY,SELECT,SENSITIVE,SEPARATOR,SERIAL,SERIALIZABLE,SERVER,SESSION,SET,SHARE,SHOW,SHUTDOWN,SIGNAL,SIGNED,SIMPLE,SLAVE,SLOW,SMALLINT,SNAPSHOT,SOCKET,SOME,SONAME,SOUNDS,SOURCE,SPATIAL,SPECIFIC,SQL,SQL_AFTER_GTIDS,SQL_AFTER_MTS_GAPS,SQL_BEFORE_GTIDS,SQL_BIG_RESULT,SQL_BUFFER_RESULT,SQL_CACHE,SQL_CALC_FOUND_ROWS,SQL_NO_CACHE,SQL_SMALL_RESULT,SQL_THREAD,SQL_TSI_DAY,SQL_TSI_HOUR,SQL_TSI_MINUTE,SQL_TSI_MONTH,SQL_TSI_QUARTER,SQL_TSI_SECOND,SQL_TSI_WEEK,SQL_TSI_YEAR,SQLEXCEPTION,SQLSTATE,SQLWARNING,SSL,STACKED,START,STARTING,STARTS,STATS_AUTO_RECALC,STATS_PERSISTENT,STATS_SAMPLE_PAGES,STATUS,STOP,STORAGE,STORED,STRAIGHT_JOIN,STRING,SUBCLASS_ORIGIN,SUBJECT,SUBPARTITION,SUBPARTITIONS,SUPER,SUSPEND,SWAPS,SWITCHES,TABLE,TABLE_CHECKSUM,TABLE_NAME,TABLES,TABLESPACE,TEMPORARY,TEMPTABLE,TERMINATED,TEXT,THAN,THEN,TIME,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TINYBLOB,TINYINT,TINYTEXT,TO,TRAILING,TRANSACTION,TRIGGER,TRIGGERS,TRUNCATE,TYPE,TYPES,UNCOMMITTED,UNDEFINED,UNDO,UNDO_BUFFER_SIZE,UNDOFILE,UNICODE,UNINSTALL,UNION,UNIQUE,UNKNOWN,UNLOCK,UNSIGNED,UNTIL,UPDATE,UPGRADE,USAGE,USE,USE_FRM,USER,USER_RESOURCES,USING,UTC_DATE,UTC_TIME,UTC_TIMESTAMP,VALIDATION,VALUE,VALUES,VARBINARY,VARCHAR,VARCHARACTER,VARIABLES,VARYING,VIEW,VIRTUAL,WAIT,WARNINGS,WEEK,WEIGHT_STRING,WHEN,WHERE,WHILE,WITH,WITHOUT,WORK,WRAPPER,WRITE,X509,XA,XID,XML,XOR,YEAR,YEAR_MONTH,ZEROFILL,FALSE,TRUE"; + private static final String ORACLE_KEYWORDS = "ACCESS,ADD,ALL,ALTER,AND,ANY,ARRAYLEN,AS,ASC,AUDIT,BETWEEN,BY,CHAR,CHECK,CLUSTER,COLUMN,COMMENT,COMPRESS,CONNECT,CREATE,CURRENT,DATE,DECIMAL,DEFAULT,DELETE,DESC,DISTINCT,DROP,ELSE,EXCLUSIVE,EXISTS,FILE,FLOAT,FOR,FROM,GRANT,GROUP,HAVING,IDENTIFIED,IMMEDIATE,IN,INCREMENT,INDEX,INITIAL,INSERT,INTEGER,INTERSECT,INTO,IS,LEVEL,LIKE,LOCK,LONG,MAXEXTENTS,MINUS,MODE,MODIFY,NOAUDIT,NOCOMPRESS,NOT,NOTFOUND,NOWAIT,NULL,NUMBER,OF,OFFLINE,ON,ONLINE,OPTION,OR,ORDER,PCTFREE,PRIOR,PRIVILEGES,PUBLIC,RAW,RENAME,RESOURCE,REVOKE,ROW,ROWID,ROWLABEL,ROWNUM,ROWS,SELECT,SESSION,SET,SHARE,SIZE,SMALLINT,SQLBUF,START,SUCCESSFUL,SYNONYM,TABLE,THEN,TO,TRIGGER,UID,UNION,UNIQUE,UPDATE,USER,VALIDATE,VALUES,VARCHAR,VARCHAR2,VIEW,WHENEVER,WHERE,WITH"; + private static Set databaseKeywords; + private static DataBaseType currentKeywordsTpye; final static public String OB_COMPATIBLE_MODE = "obCompatibilityMode"; final static public String OB_COMPATIBLE_MODE_ORACLE = "ORACLE"; final static public String OB_COMPATIBLE_MODE_MYSQL = "MYSQL"; - public static DataBaseType DATABASE_TYPE = DataBaseType.MySql; + public static DataBaseType compatibleMode = DataBaseType.MySql; + + public static DataBaseType databaseType = DataBaseType.MySql; + + + private static Set keywordsFromString2HashSet(final String keywords) { + return new HashSet(Arrays.asList(keywords.split(","))); + } + + public static void transferDatabaseKeywords(List keywords) { + //判断是否需要更改关键字集合 + if (databaseKeywords == null || currentKeywordsTpye != compatibleMode) { + if (isOracleMode(compatibleMode.toString())) { + databaseKeywords = keywordsFromString2HashSet(ORACLE_KEYWORDS); + } else { + databaseKeywords = keywordsFromString2HashSet(MYSQL_KEYWORDS); + } + currentKeywordsTpye = compatibleMode; + } + char escapeChar = isOracleMode(compatibleMode.toString()) ? '"' : '`'; + for (int i = 0; i < keywords.size(); i++) { + String keyword = keywords.get(i).toUpperCase(); + if (databaseKeywords.contains(keyword)) { + keyword = escapeChar + keyword + escapeChar; + } + keyword = keyword.toLowerCase(); + keywords.set(i, keyword); + } + } public static void initConn4Reader(Connection conn, long queryTimeoutSeconds) { String setQueryTimeout = "set ob_query_timeout=" + (queryTimeoutSeconds * 1000 * 1000L); @@ -57,7 +84,7 @@ public class ObReaderUtils { stmt = conn.createStatement(); stmt.execute(setQueryTimeout); stmt.execute(setTrxTimeout); - LOG.warn("setAutoCommit=true;"+setQueryTimeout+";"+setTrxTimeout+";"); + LOG.warn("setAutoCommit=true;" + setQueryTimeout + ";" + setTrxTimeout + ";"); } catch (Throwable e) { LOG.warn("initConn4Reader fail", e); } finally { @@ -73,7 +100,6 @@ public class ObReaderUtils { } /** - * * @param conn * @param context */ @@ -115,9 +141,9 @@ public class ObReaderUtils { String sql = "show index from " + tableName + " where Key_name='PRIMARY'"; if (isOracleMode(context.getCompatibleMode())) { tableName = tableName.toUpperCase(); - sql = "SELECT cols.column_name Column_name "+ + sql = "SELECT cols.column_name Column_name " + "FROM all_constraints cons, all_cons_columns cols " + - "WHERE cols.table_name = '" + tableName+ "' AND cons.constraint_type = 'P' " + + "WHERE cols.table_name = '" + tableName + "' AND cons.constraint_type = 'P' " + "AND cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner"; } LOG.info("get primary key by sql: " + sql); @@ -134,12 +160,7 @@ public class ObReaderUtils { realIndex.add(columnName); } } - //fix:将主键中的关键字转义 - DatabaseKeywordTransformer.setDatabaseType(DataBaseType.MySql); - if(isOracleMode(context.getCompatibleMode())){ - DatabaseKeywordTransformer.setDatabaseType(DataBaseType.Oracle); - } - DatabaseKeywordTransformer.transferDatabaseKeywords(realIndex); + transferDatabaseKeywords(realIndex); String[] pks = new String[realIndex.size()]; realIndex.toArray(pks); @@ -166,7 +187,7 @@ public class ObReaderUtils { if (StringUtils.isNotEmpty(indexName)) { String weakReadHint = weakRead ? "+READ_CONSISTENCY(WEAK)," : "+"; sql += " /*" + weakReadHint + "index(" + context.getTable() + " " + indexName + ")*/ "; - } else if (weakRead){ + } else if (weakRead) { sql += " /*+READ_CONSISTENCY(WEAK)*/ "; } sql += StringUtils.join(context.getColumns(), ','); @@ -197,7 +218,6 @@ public class ObReaderUtils { * 增量查的SQL * * @param conn - * * @param context * @return sql */ @@ -207,8 +227,8 @@ public class ObReaderUtils { String sql = "select "; if (StringUtils.isNotEmpty(indexName)) { String weakReadHint = weakRead ? "+READ_CONSISTENCY(WEAK)," : "+"; - sql += " /*"+ weakReadHint + "index(" + context.getTable() + " " + indexName + ")*/ "; - } else if (weakRead){ + sql += " /*" + weakReadHint + "index(" + context.getTable() + " " + indexName + ")*/ "; + } else if (weakRead) { sql += " /*+READ_CONSISTENCY(WEAK)*/ "; } sql += StringUtils.join(context.getColumns(), ',') + " from " + context.getTable(); @@ -305,7 +325,7 @@ public class ObReaderUtils { final char rightBracket = ')'; if (str != null && str.contains(String.valueOf(leftBracket)) && str.contains(String.valueOf(rightBracket)) && str.indexOf(leftBracket) < str.indexOf(rightBracket)) { - return str.substring(str.indexOf(leftBracket)+1, str.indexOf(rightBracket)); + return str.substring(str.indexOf(leftBracket) + 1, str.indexOf(rightBracket)); } return str; } @@ -374,7 +394,7 @@ public class ObReaderUtils { /** * 找出where条件中的列名，目前仅支持全部为and条件，并且操作符为大于、大约等于、等于、小于、小于等于和不等于的表达式。 - * + *

* test coverage: - c6 = 20180710 OR c4 = 320: no index selected - 20180710 * = c6: correct index selected - 20180710 = c6 and c4 = 320 or c2 < 100: no * index selected @@ -426,17 +446,17 @@ public class ObReaderUtils { if (isOracleMode(compatibleMode)) { tableName = tableName.toUpperCase(); sql = "SELECT INDEX_NAME Key_name, COLUMN_NAME Column_name " + - "from dba_ind_columns where TABLE_NAME = '" + tableName +"' " + + "from dba_ind_columns where TABLE_NAME = '" + tableName + "' " + " union all " + - "SELECT DISTINCT " + - "CASE " + - "WHEN cons.CONSTRAINT_TYPE = 'P' THEN 'PRIMARY' " + - "WHEN cons.CONSTRAINT_TYPE = 'U' THEN cons.CONSTRAINT_NAME " + - "ELSE '' " + - "END AS Key_name, " + - "cols.column_name Column_name " + - "FROM all_constraints cons, all_cons_columns cols " + - "WHERE cols.table_name = '" + tableName + "' AND cons.constraint_type in('P', 'U') " + + "SELECT DISTINCT " + + "CASE " + + "WHEN cons.CONSTRAINT_TYPE = 'P' THEN 'PRIMARY' " + + "WHEN cons.CONSTRAINT_TYPE = 'U' THEN cons.CONSTRAINT_NAME " + + "ELSE '' " + + "END AS Key_name, " + + "cols.column_name Column_name " + + "FROM all_constraints cons, all_cons_columns cols " + + "WHERE cols.table_name = '" + tableName + "' AND cons.constraint_type in('P', 'U') " + "AND cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner"; } Statement stmt = null; @@ -479,14 +499,13 @@ public class ObReaderUtils { } /** - * * @param conn * @param table * @param colNamesInCondition * @return */ private static List getIndexName(Connection conn, String table, - Set colNamesInCondition, String compatibleMode) { + Set colNamesInCondition, String compatibleMode) { List indexNames = new ArrayList(); if (colNamesInCondition == null || colNamesInCondition.size() == 0) { LOG.info("there is no qulified conditions in the where clause, skip index selection."); @@ -550,7 +569,7 @@ public class ObReaderUtils { Map index = new TreeMap(); List columnList = allIndexInTab.get(indexName); for (int i = 1; i <= columnList.size(); i++) { - index.put(i, columnList.get(i-1)); + index.put(i, columnList.get(i - 1)); } allIndexs.put(indexName, index); } else { @@ -654,19 +673,19 @@ public class ObReaderUtils { public static void binding(PreparedStatement ps, List list) throws SQLException { for (int i = 0, n = list.size(); i < n; i++) { Column c = list.get(i); - if(c instanceof BoolColumn){ - ps.setLong(i + 1, ((BoolColumn)c).asLong()); - }else if(c instanceof BytesColumn){ - ps.setBytes(i + 1, ((BytesColumn)c).asBytes()); - }else if(c instanceof DateColumn){ - ps.setTimestamp(i + 1, new Timestamp(((DateColumn)c).asDate().getTime())); - }else if(c instanceof DoubleColumn){ - ps.setDouble(i + 1, ((DoubleColumn)c).asDouble()); - }else if(c instanceof LongColumn){ - ps.setLong(i + 1, ((LongColumn)c).asLong()); - }else if(c instanceof StringColumn){ - ps.setString(i + 1, ((StringColumn)c).asString()); - }else{ + if (c instanceof BoolColumn) { + ps.setLong(i + 1, ((BoolColumn) c).asLong()); + } else if (c instanceof BytesColumn) { + ps.setBytes(i + 1, ((BytesColumn) c).asBytes()); + } else if (c instanceof DateColumn) { + ps.setTimestamp(i + 1, new Timestamp(((DateColumn) c).asDate().getTime())); + } else if (c instanceof DoubleColumn) { + ps.setDouble(i + 1, ((DoubleColumn) c).asDouble()); + } else if (c instanceof LongColumn) { + ps.setLong(i + 1, ((LongColumn) c).asLong()); + } else if (c instanceof StringColumn) { + ps.setString(i + 1, ((StringColumn) c).asString()); + } else { ps.setObject(i + 1, c.getRawData()); } } @@ -702,6 +721,6 @@ public class ObReaderUtils { } public static boolean isOracleMode(String mode) { - return (mode != null && OB_COMPATIBLE_MODE_ORACLE.equals(mode)); + return (mode != null && OB_COMPATIBLE_MODE_ORACLE.equals(mode.toString().toUpperCase())); } } diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/TaskContext.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/TaskContext.java index ba754a3..17655a5 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/TaskContext.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/TaskContext.java @@ -162,6 +162,7 @@ public class TaskContext { public String getUserSavePoint() { return userSavePoint; } + public void setUserSavePoint(String userSavePoint) { this.userSavePoint = userSavePoint; } From 55351fcbaaa94b600f7776667fee71f94e4057f2 Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Fri, 24 Dec 2021 10:02:17 +0800 Subject: [PATCH 079/114] =?UTF-8?q?=E5=90=8C=E6=97=B6=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=BA=86reader=E5=92=8Cwriter=EF=BC=8C=E5=90=8C=E6=97=B6?= =?UTF-8?q?=E5=B0=86DatabaseKeywordTransformer=E7=B1=BB=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=90=88=E5=B9=B6=E5=88=B0writer(reader)util?= =?UTF-8?q?s=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/OceanBaseReader.java | 5 +- .../oceanbasev10reader/ext/ReaderTask.java | 6 +- .../util/DatabaseKeywordTransformer.java | 62 ------------------- .../util/ObReaderUtils.java | 6 +- .../OceanBaseV10Writer.java | 14 +++++ .../util/ObWriterUtils.java | 31 +++++++++- 6 files changed, 51 insertions(+), 73 deletions(-) delete mode 100644 oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java index fc46159..228af81 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/OceanBaseReader.java @@ -46,7 +46,7 @@ public class OceanBaseReader extends Reader { @Override public void preCheck() { init(); - this.readerJob.preCheck(this.originalConfig, ObReaderUtils.compatibleMode); + this.readerJob.preCheck(this.originalConfig, ObReaderUtils.databaseType); } @@ -87,8 +87,7 @@ public class OceanBaseReader extends Reader { Connection conn = DBUtil.getConnection(DataBaseType.OceanBase, obJdbcUrl, username, password); String compatibleMode = ObReaderUtils.getCompatibleMode(conn); if (ObReaderUtils.isOracleMode(compatibleMode)) { - ObReaderUtils.compatibleMode = DataBaseType.Oracle; - ObReaderUtils.databaseType = DataBaseType.OceanBase; + ObReaderUtils.compatibleMode = ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; } } catch (Exception e) { diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java index b5314ad..fab0b1f 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java @@ -41,7 +41,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { private boolean reuseConn = false; public ReaderTask(int taskGroupId, int taskId) { - super(ObReaderUtils.compatibleMode, taskGroupId, taskId); + super(ObReaderUtils.databaseType, taskGroupId, taskId); this.taskGroupId = taskGroupId; this.taskId = taskId; } @@ -184,7 +184,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { } } catch (Throwable e) { if (retryLimit == ++retryCount) { - throw RdbmsException.asQueryException(ObReaderUtils.compatibleMode, new Exception(e), + throw RdbmsException.asQueryException(ObReaderUtils.databaseType, new Exception(e), context.getQuerySql(), context.getTable(), username); } LOG.error("read fail, retry count " + retryCount + ", sleep 60 second, save point:" + @@ -287,7 +287,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { ObReaderUtils.close(null, null, context.getConn()); context.setConn(null); LOG.error("reader data fail", e); - throw RdbmsException.asQueryException(ObReaderUtils.compatibleMode, e, context.getQuerySql(), + throw RdbmsException.asQueryException(ObReaderUtils.databaseType, e, context.getQuerySql(), context.getTable(), username); } finally { perfRecord.end(); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java deleted file mode 100644 index 3c3e03f..0000000 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/DatabaseKeywordTransformer.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.alibaba.datax.plugin.reader.oceanbasev10reader.util; - -//java api - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -//dataX api -import com.alibaba.datax.plugin.rdbms.util.DataBaseType; - -/** - * 该类用于转义数据库中的关键字 - * - * @author:qianzhang - */ -public class DatabaseKeywordTransformer { - private static DataBaseType databaseType; - static Set databaseKeywords; - - private static Set keywordsFromString2HashSet(final String keywords) { - return new HashSet(Arrays.asList(keywords.split(","))); - } - - public static void setDatabaseType(final DataBaseType databaseType) throws Exception { - if (databaseType == DatabaseKeywordTransformer.databaseType) { - return; - } - DatabaseKeywordTransformer.databaseType = databaseType; - if (databaseType == DataBaseType.MySql) { - databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.MYSQL_KEYWORDS); - } else if (databaseType == DataBaseType.Oracle || databaseType == DataBaseType.OceanBase) { - databaseKeywords = keywordsFromString2HashSet(DatabaseKeywords.ORACLE_KEYWORDS); - } else { - throw new Exception("sorry,unknown database tpye..."); - } - } - - public static void transferDatabaseKeywords(List keywords) { - for (int i = 0; i < keywords.size(); i++) { - String keyword = keywords.get(i).toUpperCase(); - if (databaseKeywords.contains(keyword)) { - if (databaseType == DataBaseType.MySql) { - keyword = '`' + keyword + '`'; - } else if (databaseType == DataBaseType.Oracle || databaseType == DataBaseType.OceanBase) { - keyword = '"' + keyword + '"'; - } - } - keyword = keyword.toLowerCase(); - keywords.set(i, keyword); - } - } -} - - -final class DatabaseKeywords { - - public static final String MYSQL_KEYWORDS = "ACCESSIBLE,ACCOUNT,ACTION,ADD,AFTER,AGAINST,AGGREGATE,ALGORITHM,ALL,ALTER,ALWAYS,ANALYSE,AND,ANY,AS,ASC,ASCII,ASENSITIVE,AT,AUTO_INCREMENT,AUTOEXTEND_SIZE,AVG,AVG_ROW_LENGTH,BACKUP,BEFORE,BEGIN,BETWEEN,BIGINT,BINARY,BINLOG,BIT,BLOB,BLOCK,BOOL,BOOLEAN,BOTH,BTREE,BY,BYTE,CACHE,CALL,CASCADE,CASCADED,CASE,CATALOG_NAME,CHAIN,CHANGE,CHANGED,CHANNEL,CHAR,CHARACTER,CHARSET,CHECK,CHECKSUM,CIPHER,CLASS_ORIGIN,CLIENT,CLOSE,COALESCE,CODE,COLLATE,COLLATION,COLUMN,COLUMN_FORMAT,COLUMN_NAME,COLUMNS,COMMENT,COMMIT,COMMITTED,COMPACT,COMPLETION,COMPRESSED,COMPRESSION,CONCURRENT,CONDITION,CONNECTION,CONSISTENT,CONSTRAINT,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CONTAINS,CONTEXT,CONTINUE,CONVERT,CPU,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,CURSOR_NAME,DATA,DATABASE,DATABASES,DATAFILE,DATE,DATETIME,DAY,DAY_HOUR,DAY_MICROSECOND,DAY_MINUTE,DAY_SECOND,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_AUTH,DEFINER,DELAY_KEY_WRITE,DELAYED,DELETE,DES_KEY_FILE,DESC,DESCRIBE,DETERMINISTIC,DIAGNOSTICS,DIRECTORY,DISABLE,DISCARD,DISK,DISTINCT,DISTINCTROW,DIV,DO,DOUBLE,DROP,DUAL,DUMPFILE,DUPLICATE,DYNAMIC,EACH,ELSE,ELSEIF,ENABLE,ENCLOSED,ENCRYPTION,END,ENDS,ENGINE,ENGINES,ENUM,ERROR,ERRORS,ESCAPE,ESCAPED,EVENT,EVENTS,EVERY,EXCHANGE,EXECUTE,EXISTS,EXIT,EXPANSION,EXPIRE,EXPLAIN,EXPORT,EXTENDED,EXTENT_SIZE,FAST,FAULTS,FETCH,FIELDS,FILE,FILE_BLOCK_SIZE,FILTER,FIRST,FIXED,FLOAT,FLOAT4,FLOAT8,FLUSH,FOLLOWS,FOR,FORCE,FOREIGN,FORMAT,FOUND,FROM,FULL,FULLTEXT,FUNCTION,GENERAL,GENERATED,GEOMETRY,GEOMETRYCOLLECTION,GET,GET_FORMAT,GLOBAL,GRANT,GRANTS,GROUP,GROUP_REPLICATION,HANDLER,HASH,HAVING,HELP,HIGH_PRIORITY,HOST,HOSTS,HOUR,HOUR_MICROSECOND,HOUR_MINUTE,HOUR_SECOND,IDENTIFIED,IF,IGNORE,IGNORE_SERVER_IDS,IMPORT,IN,INDEX,INDEXES,INFILE,INITIAL_SIZE,INNER,INOUT,INSENSITIVE,INSERT,INSERT_METHOD,INSTALL,INSTANCE,INT,INT1,INT2,INT3,INT4,INT8,INTEGER,INTERVAL,INTO,INVOKER,IO,IO_AFTER_GTIDS,IO_BEFORE_GTIDS,IO_THREAD,IPC,IS,ISOLATION,ISSUER,ITERATE,JOIN,JSON,KEY,KEY_BLOCK_SIZE,KEYS,KILL,LANGUAGE,LAST,LEADING,LEAVE,LEAVES,LEFT,LESS,LEVEL,LIKE,LIMIT,LINEAR,LINES,LINESTRING,LIST,LOAD,LOCAL,LOCALTIME,LOCALTIMESTAMP,LOCK,LOCKS,LOGFILE,LOGS,LONG,LONGBLOB,LONGTEXT,LOOP,LOW_PRIORITY,MASTER,MASTER_AUTO_POSITION,MASTER_BIND,MASTER_CONNECT_RETRY,MASTER_DELAY,MASTER_HEARTBEAT_PERIOD,MASTER_HOST,MASTER_LOG_FILE,MASTER_LOG_POS,MASTER_PASSWORD,MASTER_PORT,MASTER_RETRY_COUNT,MASTER_SERVER_ID,MASTER_SSL,MASTER_SSL_CA,MASTER_SSL_CAPATH,MASTER_SSL_CERT,MASTER_SSL_CIPHER,MASTER_SSL_CRL,MASTER_SSL_CRLPATH,MASTER_SSL_KEY,MASTER_SSL_VERIFY_SERVER_CERT,MASTER_TLS_VERSION,MASTER_USER,MATCH,MAX_CONNECTIONS_PER_HOUR,MAX_QUERIES_PER_HOUR,MAX_ROWS,MAX_SIZE,MAX_STATEMENT_TIME,MAX_UPDATES_PER_HOUR,MAX_USER_CONNECTIONS,MAXVALUE,MEDIUM,MEDIUMBLOB,MEDIUMINT,MEDIUMTEXT,MEMORY,MERGE,MESSAGE_TEXT,MICROSECOND,MIDDLEINT,MIGRATE,MIN_ROWS,MINUTE,MINUTE_MICROSECOND,MINUTE_SECOND,MOD,MODE,MODIFIES,MODIFY,MONTH,MULTILINESTRING,MULTIPOINT,MULTIPOLYGON,MUTEX,MYSQL_ERRNO,NAME,NAMES,NATIONAL,NATURAL,NCHAR,NDB,NDBCLUSTER,NEVER,NEW,NEXT,NO,NO_WAIT,NO_WRITE_TO_BINLOG,NODEGROUP,NONBLOCKING,NONE,NOT,NULL,NUMBER,NUMERIC,NVARCHAR,OFFSET,OLD_PASSWORD,ON,ONE,ONLY,OPEN,OPTIMIZE,OPTIMIZER_COSTS,OPTION,OPTIONALLY,OPTIONS,OR,ORDER,OUT,OUTER,OUTFILE,OWNER,PACK_KEYS,PAGE,PARSE_GCOL_EXPR,PARSER,PARTIAL,PARTITION,PARTITIONING,PARTITIONS,PASSWORD,PHASE,PLUGIN,PLUGIN_DIR,PLUGINS,POINT,POLYGON,PORT,PRECEDES,PRECISION,PREPARE,PRESERVE,PREV,PRIMARY,PRIVILEGES,PROCEDURE,PROCESSLIST,PROFILE,PROFILES,PROXY,PURGE,QUARTER,QUERY,QUICK,RANGE,READ,READ_ONLY,READ_WRITE,READS,REAL,REBUILD,RECOVER,REDO_BUFFER_SIZE,REDOFILE,REDUNDANT,REFERENCES,REGEXP,RELAY,RELAY_LOG_FILE,RELAY_LOG_POS,RELAY_THREAD,RELAYLOG,RELEASE,RELOAD,REMOVE,RENAME,REORGANIZE,REPAIR,REPEAT,REPEATABLE,REPLACE,REPLICATE_DO_DB,REPLICATE_DO_TABLE,REPLICATE_IGNORE_DB,REPLICATE_IGNORE_TABLE,REPLICATE_REWRITE_DB,REPLICATE_WILD_DO_TABLE,REPLICATE_WILD_IGNORE_TABLE,REPLICATION,REQUIRE,RESET,RESIGNAL,RESTORE,RESTRICT,RESUME,RETURN,RETURNED_SQLSTATE,RETURNS,REVERSE,REVOKE,RIGHT,RLIKE,ROLLBACK,ROLLUP,ROTATE,ROUTINE,ROW,ROW_COUNT,ROW_FORMAT,ROWS,RTREE,SAVEPOINT,SCHEDULE,SCHEMA,SCHEMA_NAME,SCHEMAS,SECOND,SECOND_MICROSECOND,SECURITY,SELECT,SENSITIVE,SEPARATOR,SERIAL,SERIALIZABLE,SERVER,SESSION,SET,SHARE,SHOW,SHUTDOWN,SIGNAL,SIGNED,SIMPLE,SLAVE,SLOW,SMALLINT,SNAPSHOT,SOCKET,SOME,SONAME,SOUNDS,SOURCE,SPATIAL,SPECIFIC,SQL,SQL_AFTER_GTIDS,SQL_AFTER_MTS_GAPS,SQL_BEFORE_GTIDS,SQL_BIG_RESULT,SQL_BUFFER_RESULT,SQL_CACHE,SQL_CALC_FOUND_ROWS,SQL_NO_CACHE,SQL_SMALL_RESULT,SQL_THREAD,SQL_TSI_DAY,SQL_TSI_HOUR,SQL_TSI_MINUTE,SQL_TSI_MONTH,SQL_TSI_QUARTER,SQL_TSI_SECOND,SQL_TSI_WEEK,SQL_TSI_YEAR,SQLEXCEPTION,SQLSTATE,SQLWARNING,SSL,STACKED,START,STARTING,STARTS,STATS_AUTO_RECALC,STATS_PERSISTENT,STATS_SAMPLE_PAGES,STATUS,STOP,STORAGE,STORED,STRAIGHT_JOIN,STRING,SUBCLASS_ORIGIN,SUBJECT,SUBPARTITION,SUBPARTITIONS,SUPER,SUSPEND,SWAPS,SWITCHES,TABLE,TABLE_CHECKSUM,TABLE_NAME,TABLES,TABLESPACE,TEMPORARY,TEMPTABLE,TERMINATED,TEXT,THAN,THEN,TIME,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TINYBLOB,TINYINT,TINYTEXT,TO,TRAILING,TRANSACTION,TRIGGER,TRIGGERS,TRUNCATE,TYPE,TYPES,UNCOMMITTED,UNDEFINED,UNDO,UNDO_BUFFER_SIZE,UNDOFILE,UNICODE,UNINSTALL,UNION,UNIQUE,UNKNOWN,UNLOCK,UNSIGNED,UNTIL,UPDATE,UPGRADE,USAGE,USE,USE_FRM,USER,USER_RESOURCES,USING,UTC_DATE,UTC_TIME,UTC_TIMESTAMP,VALIDATION,VALUE,VALUES,VARBINARY,VARCHAR,VARCHARACTER,VARIABLES,VARYING,VIEW,VIRTUAL,WAIT,WARNINGS,WEEK,WEIGHT_STRING,WHEN,WHERE,WHILE,WITH,WITHOUT,WORK,WRAPPER,WRITE,X509,XA,XID,XML,XOR,YEAR,YEAR_MONTH,ZEROFILL,FALSE,TRUE"; - - public static final String ORACLE_KEYWORDS = "ACCESS,ADD,ALL,ALTER,AND,ANY,ARRAYLEN,AS,ASC,AUDIT,BETWEEN,BY,CHAR,CHECK,CLUSTER,COLUMN,COMMENT,COMPRESS,CONNECT,CREATE,CURRENT,DATE,DECIMAL,DEFAULT,DELETE,DESC,DISTINCT,DROP,ELSE,EXCLUSIVE,EXISTS,FILE,FLOAT,FOR,FROM,GRANT,GROUP,HAVING,IDENTIFIED,IMMEDIATE,IN,INCREMENT,INDEX,INITIAL,INSERT,INTEGER,INTERSECT,INTO,IS,LEVEL,LIKE,LOCK,LONG,MAXEXTENTS,MINUS,MODE,MODIFY,NOAUDIT,NOCOMPRESS,NOT,NOTFOUND,NOWAIT,NULL,NUMBER,OF,OFFLINE,ON,ONLINE,OPTION,OR,ORDER,PCTFREE,PRIOR,PRIVILEGES,PUBLIC,RAW,RENAME,RESOURCE,REVOKE,ROW,ROWID,ROWLABEL,ROWNUM,ROWS,SELECT,SESSION,SET,SHARE,SIZE,SMALLINT,SQLBUF,START,SUCCESSFUL,SYNONYM,TABLE,THEN,TO,TRIGGER,UID,UNION,UNIQUE,UPDATE,USER,VALIDATE,VALUES,VARCHAR,VARCHAR2,VIEW,WHENEVER,WHERE,WITH"; -} \ No newline at end of file diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java index 5084523..8efee64 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java @@ -40,14 +40,14 @@ public class ObReaderUtils { private static final String ORACLE_KEYWORDS = "ACCESS,ADD,ALL,ALTER,AND,ANY,ARRAYLEN,AS,ASC,AUDIT,BETWEEN,BY,CHAR,CHECK,CLUSTER,COLUMN,COMMENT,COMPRESS,CONNECT,CREATE,CURRENT,DATE,DECIMAL,DEFAULT,DELETE,DESC,DISTINCT,DROP,ELSE,EXCLUSIVE,EXISTS,FILE,FLOAT,FOR,FROM,GRANT,GROUP,HAVING,IDENTIFIED,IMMEDIATE,IN,INCREMENT,INDEX,INITIAL,INSERT,INTEGER,INTERSECT,INTO,IS,LEVEL,LIKE,LOCK,LONG,MAXEXTENTS,MINUS,MODE,MODIFY,NOAUDIT,NOCOMPRESS,NOT,NOTFOUND,NOWAIT,NULL,NUMBER,OF,OFFLINE,ON,ONLINE,OPTION,OR,ORDER,PCTFREE,PRIOR,PRIVILEGES,PUBLIC,RAW,RENAME,RESOURCE,REVOKE,ROW,ROWID,ROWLABEL,ROWNUM,ROWS,SELECT,SESSION,SET,SHARE,SIZE,SMALLINT,SQLBUF,START,SUCCESSFUL,SYNONYM,TABLE,THEN,TO,TRIGGER,UID,UNION,UNIQUE,UPDATE,USER,VALIDATE,VALUES,VARCHAR,VARCHAR2,VIEW,WHENEVER,WHERE,WITH"; private static Set databaseKeywords; - private static DataBaseType currentKeywordsTpye; + private static String currentKeywordsTpye; final static public String OB_COMPATIBLE_MODE = "obCompatibilityMode"; final static public String OB_COMPATIBLE_MODE_ORACLE = "ORACLE"; final static public String OB_COMPATIBLE_MODE_MYSQL = "MYSQL"; - public static DataBaseType compatibleMode = DataBaseType.MySql; + public static String compatibleMode = OB_COMPATIBLE_MODE_MYSQL; - public static DataBaseType databaseType = DataBaseType.MySql; + public static final DataBaseType databaseType = DataBaseType.OceanBase; private static Set keywordsFromString2HashSet(final String keywords) { diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java index 4ffaffe..bf1c9f6 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.DbUtils; +import com.alibaba.fastjson.JSONObject; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +61,19 @@ public class OceanBaseV10Writer extends Writer { public void init() { this.originalConfig = super.getPluginJobConf(); checkCompatibleMode(originalConfig); + //将config中的column和table中的关键字进行转义 + List columns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Key.COLUMN, String.class); + ObWriterUtils.transferDatabaseKeywords(columns); + originalConfig.set(com.alibaba.datax.plugin.rdbms.reader.Key.COLUMN, columns); + + List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, JSONObject.class); + for (int i = 0; i < conns.size(); i++) { + JSONObject conn = conns.get(i); + Configuration connConfig = Configuration.from(conn.toString()); + List tables = connConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Key.TABLE, String.class); + ObWriterUtils.transferDatabaseKeywords(tables); + originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, i, com.alibaba.datax.plugin.rdbms.reader.Key.TABLE), tables); + } this.commonJob = new CommonRdbmsWriter.Job(DATABASE_TYPE); this.commonJob.init(this.originalConfig); } diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java index 368c3d1..a8ff7a0 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java @@ -14,12 +14,39 @@ import java.util.*; import java.util.stream.Collectors; public class ObWriterUtils { - protected static final Logger LOG = LoggerFactory.getLogger(Task.class); + + private static final String MYSQL_KEYWORDS = "ACCESSIBLE,ACCOUNT,ACTION,ADD,AFTER,AGAINST,AGGREGATE,ALGORITHM,ALL,ALTER,ALWAYS,ANALYSE,AND,ANY,AS,ASC,ASCII,ASENSITIVE,AT,AUTO_INCREMENT,AUTOEXTEND_SIZE,AVG,AVG_ROW_LENGTH,BACKUP,BEFORE,BEGIN,BETWEEN,BIGINT,BINARY,BINLOG,BIT,BLOB,BLOCK,BOOL,BOOLEAN,BOTH,BTREE,BY,BYTE,CACHE,CALL,CASCADE,CASCADED,CASE,CATALOG_NAME,CHAIN,CHANGE,CHANGED,CHANNEL,CHAR,CHARACTER,CHARSET,CHECK,CHECKSUM,CIPHER,CLASS_ORIGIN,CLIENT,CLOSE,COALESCE,CODE,COLLATE,COLLATION,COLUMN,COLUMN_FORMAT,COLUMN_NAME,COLUMNS,COMMENT,COMMIT,COMMITTED,COMPACT,COMPLETION,COMPRESSED,COMPRESSION,CONCURRENT,CONDITION,CONNECTION,CONSISTENT,CONSTRAINT,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CONTAINS,CONTEXT,CONTINUE,CONVERT,CPU,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,CURSOR_NAME,DATA,DATABASE,DATABASES,DATAFILE,DATE,DATETIME,DAY,DAY_HOUR,DAY_MICROSECOND,DAY_MINUTE,DAY_SECOND,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_AUTH,DEFINER,DELAY_KEY_WRITE,DELAYED,DELETE,DES_KEY_FILE,DESC,DESCRIBE,DETERMINISTIC,DIAGNOSTICS,DIRECTORY,DISABLE,DISCARD,DISK,DISTINCT,DISTINCTROW,DIV,DO,DOUBLE,DROP,DUAL,DUMPFILE,DUPLICATE,DYNAMIC,EACH,ELSE,ELSEIF,ENABLE,ENCLOSED,ENCRYPTION,END,ENDS,ENGINE,ENGINES,ENUM,ERROR,ERRORS,ESCAPE,ESCAPED,EVENT,EVENTS,EVERY,EXCHANGE,EXECUTE,EXISTS,EXIT,EXPANSION,EXPIRE,EXPLAIN,EXPORT,EXTENDED,EXTENT_SIZE,FAST,FAULTS,FETCH,FIELDS,FILE,FILE_BLOCK_SIZE,FILTER,FIRST,FIXED,FLOAT,FLOAT4,FLOAT8,FLUSH,FOLLOWS,FOR,FORCE,FOREIGN,FORMAT,FOUND,FROM,FULL,FULLTEXT,FUNCTION,GENERAL,GENERATED,GEOMETRY,GEOMETRYCOLLECTION,GET,GET_FORMAT,GLOBAL,GRANT,GRANTS,GROUP,GROUP_REPLICATION,HANDLER,HASH,HAVING,HELP,HIGH_PRIORITY,HOST,HOSTS,HOUR,HOUR_MICROSECOND,HOUR_MINUTE,HOUR_SECOND,IDENTIFIED,IF,IGNORE,IGNORE_SERVER_IDS,IMPORT,IN,INDEX,INDEXES,INFILE,INITIAL_SIZE,INNER,INOUT,INSENSITIVE,INSERT,INSERT_METHOD,INSTALL,INSTANCE,INT,INT1,INT2,INT3,INT4,INT8,INTEGER,INTERVAL,INTO,INVOKER,IO,IO_AFTER_GTIDS,IO_BEFORE_GTIDS,IO_THREAD,IPC,IS,ISOLATION,ISSUER,ITERATE,JOIN,JSON,KEY,KEY_BLOCK_SIZE,KEYS,KILL,LANGUAGE,LAST,LEADING,LEAVE,LEAVES,LEFT,LESS,LEVEL,LIKE,LIMIT,LINEAR,LINES,LINESTRING,LIST,LOAD,LOCAL,LOCALTIME,LOCALTIMESTAMP,LOCK,LOCKS,LOGFILE,LOGS,LONG,LONGBLOB,LONGTEXT,LOOP,LOW_PRIORITY,MASTER,MASTER_AUTO_POSITION,MASTER_BIND,MASTER_CONNECT_RETRY,MASTER_DELAY,MASTER_HEARTBEAT_PERIOD,MASTER_HOST,MASTER_LOG_FILE,MASTER_LOG_POS,MASTER_PASSWORD,MASTER_PORT,MASTER_RETRY_COUNT,MASTER_SERVER_ID,MASTER_SSL,MASTER_SSL_CA,MASTER_SSL_CAPATH,MASTER_SSL_CERT,MASTER_SSL_CIPHER,MASTER_SSL_CRL,MASTER_SSL_CRLPATH,MASTER_SSL_KEY,MASTER_SSL_VERIFY_SERVER_CERT,MASTER_TLS_VERSION,MASTER_USER,MATCH,MAX_CONNECTIONS_PER_HOUR,MAX_QUERIES_PER_HOUR,MAX_ROWS,MAX_SIZE,MAX_STATEMENT_TIME,MAX_UPDATES_PER_HOUR,MAX_USER_CONNECTIONS,MAXVALUE,MEDIUM,MEDIUMBLOB,MEDIUMINT,MEDIUMTEXT,MEMORY,MERGE,MESSAGE_TEXT,MICROSECOND,MIDDLEINT,MIGRATE,MIN_ROWS,MINUTE,MINUTE_MICROSECOND,MINUTE_SECOND,MOD,MODE,MODIFIES,MODIFY,MONTH,MULTILINESTRING,MULTIPOINT,MULTIPOLYGON,MUTEX,MYSQL_ERRNO,NAME,NAMES,NATIONAL,NATURAL,NCHAR,NDB,NDBCLUSTER,NEVER,NEW,NEXT,NO,NO_WAIT,NO_WRITE_TO_BINLOG,NODEGROUP,NONBLOCKING,NONE,NOT,NULL,NUMBER,NUMERIC,NVARCHAR,OFFSET,OLD_PASSWORD,ON,ONE,ONLY,OPEN,OPTIMIZE,OPTIMIZER_COSTS,OPTION,OPTIONALLY,OPTIONS,OR,ORDER,OUT,OUTER,OUTFILE,OWNER,PACK_KEYS,PAGE,PARSE_GCOL_EXPR,PARSER,PARTIAL,PARTITION,PARTITIONING,PARTITIONS,PASSWORD,PHASE,PLUGIN,PLUGIN_DIR,PLUGINS,POINT,POLYGON,PORT,PRECEDES,PRECISION,PREPARE,PRESERVE,PREV,PRIMARY,PRIVILEGES,PROCEDURE,PROCESSLIST,PROFILE,PROFILES,PROXY,PURGE,QUARTER,QUERY,QUICK,RANGE,READ,READ_ONLY,READ_WRITE,READS,REAL,REBUILD,RECOVER,REDO_BUFFER_SIZE,REDOFILE,REDUNDANT,REFERENCES,REGEXP,RELAY,RELAY_LOG_FILE,RELAY_LOG_POS,RELAY_THREAD,RELAYLOG,RELEASE,RELOAD,REMOVE,RENAME,REORGANIZE,REPAIR,REPEAT,REPEATABLE,REPLACE,REPLICATE_DO_DB,REPLICATE_DO_TABLE,REPLICATE_IGNORE_DB,REPLICATE_IGNORE_TABLE,REPLICATE_REWRITE_DB,REPLICATE_WILD_DO_TABLE,REPLICATE_WILD_IGNORE_TABLE,REPLICATION,REQUIRE,RESET,RESIGNAL,RESTORE,RESTRICT,RESUME,RETURN,RETURNED_SQLSTATE,RETURNS,REVERSE,REVOKE,RIGHT,RLIKE,ROLLBACK,ROLLUP,ROTATE,ROUTINE,ROW,ROW_COUNT,ROW_FORMAT,ROWS,RTREE,SAVEPOINT,SCHEDULE,SCHEMA,SCHEMA_NAME,SCHEMAS,SECOND,SECOND_MICROSECOND,SECURITY,SELECT,SENSITIVE,SEPARATOR,SERIAL,SERIALIZABLE,SERVER,SESSION,SET,SHARE,SHOW,SHUTDOWN,SIGNAL,SIGNED,SIMPLE,SLAVE,SLOW,SMALLINT,SNAPSHOT,SOCKET,SOME,SONAME,SOUNDS,SOURCE,SPATIAL,SPECIFIC,SQL,SQL_AFTER_GTIDS,SQL_AFTER_MTS_GAPS,SQL_BEFORE_GTIDS,SQL_BIG_RESULT,SQL_BUFFER_RESULT,SQL_CACHE,SQL_CALC_FOUND_ROWS,SQL_NO_CACHE,SQL_SMALL_RESULT,SQL_THREAD,SQL_TSI_DAY,SQL_TSI_HOUR,SQL_TSI_MINUTE,SQL_TSI_MONTH,SQL_TSI_QUARTER,SQL_TSI_SECOND,SQL_TSI_WEEK,SQL_TSI_YEAR,SQLEXCEPTION,SQLSTATE,SQLWARNING,SSL,STACKED,START,STARTING,STARTS,STATS_AUTO_RECALC,STATS_PERSISTENT,STATS_SAMPLE_PAGES,STATUS,STOP,STORAGE,STORED,STRAIGHT_JOIN,STRING,SUBCLASS_ORIGIN,SUBJECT,SUBPARTITION,SUBPARTITIONS,SUPER,SUSPEND,SWAPS,SWITCHES,TABLE,TABLE_CHECKSUM,TABLE_NAME,TABLES,TABLESPACE,TEMPORARY,TEMPTABLE,TERMINATED,TEXT,THAN,THEN,TIME,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TINYBLOB,TINYINT,TINYTEXT,TO,TRAILING,TRANSACTION,TRIGGER,TRIGGERS,TRUNCATE,TYPE,TYPES,UNCOMMITTED,UNDEFINED,UNDO,UNDO_BUFFER_SIZE,UNDOFILE,UNICODE,UNINSTALL,UNION,UNIQUE,UNKNOWN,UNLOCK,UNSIGNED,UNTIL,UPDATE,UPGRADE,USAGE,USE,USE_FRM,USER,USER_RESOURCES,USING,UTC_DATE,UTC_TIME,UTC_TIMESTAMP,VALIDATION,VALUE,VALUES,VARBINARY,VARCHAR,VARCHARACTER,VARIABLES,VARYING,VIEW,VIRTUAL,WAIT,WARNINGS,WEEK,WEIGHT_STRING,WHEN,WHERE,WHILE,WITH,WITHOUT,WORK,WRAPPER,WRITE,X509,XA,XID,XML,XOR,YEAR,YEAR_MONTH,ZEROFILL,FALSE,TRUE"; + private static final String ORACLE_KEYWORDS = "ACCESS,ADD,ALL,ALTER,AND,ANY,ARRAYLEN,AS,ASC,AUDIT,BETWEEN,BY,CHAR,CHECK,CLUSTER,COLUMN,COMMENT,COMPRESS,CONNECT,CREATE,CURRENT,DATE,DECIMAL,DEFAULT,DELETE,DESC,DISTINCT,DROP,ELSE,EXCLUSIVE,EXISTS,FILE,FLOAT,FOR,FROM,GRANT,GROUP,HAVING,IDENTIFIED,IMMEDIATE,IN,INCREMENT,INDEX,INITIAL,INSERT,INTEGER,INTERSECT,INTO,IS,LEVEL,LIKE,LOCK,LONG,MAXEXTENTS,MINUS,MODE,MODIFY,NOAUDIT,NOCOMPRESS,NOT,NOTFOUND,NOWAIT,NULL,NUMBER,OF,OFFLINE,ON,ONLINE,OPTION,OR,ORDER,PCTFREE,PRIOR,PRIVILEGES,PUBLIC,RAW,RENAME,RESOURCE,REVOKE,ROW,ROWID,ROWLABEL,ROWNUM,ROWS,SELECT,SESSION,SET,SHARE,SIZE,SMALLINT,SQLBUF,START,SUCCESSFUL,SYNONYM,TABLE,THEN,TO,TRIGGER,UID,UNION,UNIQUE,UPDATE,USER,VALIDATE,VALUES,VARCHAR,VARCHAR2,VIEW,WHENEVER,WHERE,WITH"; private static String CHECK_MEMSTORE = "select 1 from %s.gv$memstore t where t.total>t.mem_limit * ?"; - + private static Set databaseKeywords; private static String compatibleMode = null; + private static String currentKeywordsTpye =null; + protected static final Logger LOG = LoggerFactory.getLogger(Task.class); + private static Set keywordsFromString2HashSet(final String keywords) { + return new HashSet(Arrays.asList(keywords.split(","))); + } + public static void transferDatabaseKeywords(List keywords) { + //判断是否需要更改关键字集合 + if (databaseKeywords == null || currentKeywordsTpye != compatibleMode) { + if (isOracleMode()) { + databaseKeywords = keywordsFromString2HashSet(ORACLE_KEYWORDS); + } else { + databaseKeywords = keywordsFromString2HashSet(MYSQL_KEYWORDS); + } + currentKeywordsTpye = compatibleMode; + } + char escapeChar = isOracleMode() ? '"' : '`'; + for (int i = 0; i < keywords.size(); i++) { + String keyword = keywords.get(i).toUpperCase(); + if (databaseKeywords.contains(keyword)) { + keyword = escapeChar + keyword + escapeChar; + } + keyword = keyword.toLowerCase(); + keywords.set(i, keyword); + } + } public static boolean isMemstoreFull(Connection conn, double memstoreThreshold) { PreparedStatement ps = null; ResultSet rs = null; From dcb541f04801afb9fc95f28e6aec348e216952a8 Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Mon, 27 Dec 2021 17:02:51 +0800 Subject: [PATCH 080/114] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E4=B8=8A?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E7=89=88=E6=9C=AC=E6=8F=90=E5=88=B0=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../reader/oceanbasev10reader/ext/ReaderJob.java | 4 ++-- .../reader/oceanbasev10reader/ext/ReaderTask.java | 4 +++- .../oceanbasev10reader/util/ObReaderUtils.java | 14 +++++--------- .../oceanbasev10writer/OceanBaseV10Writer.java | 15 +++++++-------- .../task/ConcurrentTableWriterTask.java | 7 ++++++- .../oceanbasev10writer/util/ObWriterUtils.java | 10 ++++------ 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java index 2a31aaa..f69a916 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderJob.java @@ -26,7 +26,7 @@ public class ReaderJob extends CommonRdbmsReader.Job { public void init(Configuration originalConfig) { //将config中的column和table中的关键字进行转义 List columns = originalConfig.getList(Key.COLUMN, String.class); - ObReaderUtils.transferDatabaseKeywords(columns); + ObReaderUtils.escapeDatabaseKeywords(columns); originalConfig.set(Key.COLUMN, columns); List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, JSONObject.class); @@ -34,7 +34,7 @@ public class ReaderJob extends CommonRdbmsReader.Job { JSONObject conn = conns.get(i); Configuration connConfig = Configuration.from(conn.toString()); List tables = connConfig.getList(Key.TABLE, String.class); - ObReaderUtils.transferDatabaseKeywords(tables); + ObReaderUtils.escapeDatabaseKeywords(tables); originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, i, Key.TABLE), tables); } super.init(originalConfig); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java index fab0b1f..254b334 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java @@ -66,10 +66,12 @@ public class ReaderTask extends CommonRdbmsReader.Task { if (ObReaderUtils.databaseType == DataBaseType.OceanBase) { jdbcUrl = jdbcUrl.replace("jdbc:mysql:", "jdbc:oceanbase:") + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 - compatibleMode = ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; } else { jdbcUrl = jdbcUrl + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 } + if(ObReaderUtils.compatibleMode==ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE){ + compatibleMode=ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; + } LOG.info("this is ob1_0 jdbc url. user=" + username + " :url=" + jdbcUrl); mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); retryLimit = readerSliceConfig.getInt(Config.RETRY_LIMIT, Config.DEFAULT_RETRY_LIMIT); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java index 8efee64..143171d 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java @@ -40,7 +40,6 @@ public class ObReaderUtils { private static final String ORACLE_KEYWORDS = "ACCESS,ADD,ALL,ALTER,AND,ANY,ARRAYLEN,AS,ASC,AUDIT,BETWEEN,BY,CHAR,CHECK,CLUSTER,COLUMN,COMMENT,COMPRESS,CONNECT,CREATE,CURRENT,DATE,DECIMAL,DEFAULT,DELETE,DESC,DISTINCT,DROP,ELSE,EXCLUSIVE,EXISTS,FILE,FLOAT,FOR,FROM,GRANT,GROUP,HAVING,IDENTIFIED,IMMEDIATE,IN,INCREMENT,INDEX,INITIAL,INSERT,INTEGER,INTERSECT,INTO,IS,LEVEL,LIKE,LOCK,LONG,MAXEXTENTS,MINUS,MODE,MODIFY,NOAUDIT,NOCOMPRESS,NOT,NOTFOUND,NOWAIT,NULL,NUMBER,OF,OFFLINE,ON,ONLINE,OPTION,OR,ORDER,PCTFREE,PRIOR,PRIVILEGES,PUBLIC,RAW,RENAME,RESOURCE,REVOKE,ROW,ROWID,ROWLABEL,ROWNUM,ROWS,SELECT,SESSION,SET,SHARE,SIZE,SMALLINT,SQLBUF,START,SUCCESSFUL,SYNONYM,TABLE,THEN,TO,TRIGGER,UID,UNION,UNIQUE,UPDATE,USER,VALIDATE,VALUES,VARCHAR,VARCHAR2,VIEW,WHENEVER,WHERE,WITH"; private static Set databaseKeywords; - private static String currentKeywordsTpye; final static public String OB_COMPATIBLE_MODE = "obCompatibilityMode"; final static public String OB_COMPATIBLE_MODE_ORACLE = "ORACLE"; final static public String OB_COMPATIBLE_MODE_MYSQL = "MYSQL"; @@ -54,23 +53,20 @@ public class ObReaderUtils { return new HashSet(Arrays.asList(keywords.split(","))); } - public static void transferDatabaseKeywords(List keywords) { - //判断是否需要更改关键字集合 - if (databaseKeywords == null || currentKeywordsTpye != compatibleMode) { + public static void escapeDatabaseKeywords(List keywords) { + if (databaseKeywords == null) { if (isOracleMode(compatibleMode.toString())) { databaseKeywords = keywordsFromString2HashSet(ORACLE_KEYWORDS); } else { databaseKeywords = keywordsFromString2HashSet(MYSQL_KEYWORDS); } - currentKeywordsTpye = compatibleMode; } char escapeChar = isOracleMode(compatibleMode.toString()) ? '"' : '`'; for (int i = 0; i < keywords.size(); i++) { - String keyword = keywords.get(i).toUpperCase(); - if (databaseKeywords.contains(keyword)) { + String keyword = keywords.get(i); + if (databaseKeywords.contains(keyword.toUpperCase())) { keyword = escapeChar + keyword + escapeChar; } - keyword = keyword.toLowerCase(); keywords.set(i, keyword); } } @@ -160,7 +156,7 @@ public class ObReaderUtils { realIndex.add(columnName); } } - transferDatabaseKeywords(realIndex); + escapeDatabaseKeywords(realIndex); String[] pks = new String[realIndex.size()]; realIndex.toArray(pks); diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java index bf1c9f6..417fccb 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java @@ -3,7 +3,6 @@ package com.alibaba.datax.plugin.writer.oceanbasev10writer; import java.sql.*; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.DbUtils; import com.alibaba.fastjson.JSONObject; @@ -62,17 +61,17 @@ public class OceanBaseV10Writer extends Writer { this.originalConfig = super.getPluginJobConf(); checkCompatibleMode(originalConfig); //将config中的column和table中的关键字进行转义 - List columns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Key.COLUMN, String.class); - ObWriterUtils.transferDatabaseKeywords(columns); - originalConfig.set(com.alibaba.datax.plugin.rdbms.reader.Key.COLUMN, columns); + List columns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.writer.Key.COLUMN, String.class); + ObWriterUtils.escapeDatabaseKeywords(columns); + originalConfig.set(com.alibaba.datax.plugin.rdbms.writer.Key.COLUMN, columns); - List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, JSONObject.class); + List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.writer.Constant.CONN_MARK, JSONObject.class); for (int i = 0; i < conns.size(); i++) { JSONObject conn = conns.get(i); Configuration connConfig = Configuration.from(conn.toString()); - List tables = connConfig.getList(com.alibaba.datax.plugin.rdbms.reader.Key.TABLE, String.class); - ObWriterUtils.transferDatabaseKeywords(tables); - originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.reader.Constant.CONN_MARK, i, com.alibaba.datax.plugin.rdbms.reader.Key.TABLE), tables); + List tables = connConfig.getList(com.alibaba.datax.plugin.rdbms.writer.Key.TABLE, String.class); + ObWriterUtils.escapeDatabaseKeywords(tables); + originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.writer.Constant.CONN_MARK, i, com.alibaba.datax.plugin.rdbms.writer.Key.TABLE), tables); } this.commonJob = new CommonRdbmsWriter.Job(DATABASE_TYPE); this.commonJob.init(this.originalConfig); diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index cbc9a93..f07c543 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -15,6 +15,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; import com.alibaba.datax.common.element.Column; import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ObClientConnHolder; @@ -105,7 +106,11 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { checkConnHolder.initConnection(); if (isOracleCompatibleMode) { connectInfo.databaseName = connectInfo.databaseName.toUpperCase(); - table = table.toUpperCase(); + //在转义的情况下不翻译 + if(!Pattern.matches("\"\\w*\"",table)){ + table = table.toUpperCase(); + } + LOG.info(String.format("this is oracle compatible mode, change database to %s, table to %s", connectInfo.databaseName, table)); } diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java index a8ff7a0..8e9b420 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java @@ -21,26 +21,24 @@ public class ObWriterUtils { private static String CHECK_MEMSTORE = "select 1 from %s.gv$memstore t where t.total>t.mem_limit * ?"; private static Set databaseKeywords; private static String compatibleMode = null; - private static String currentKeywordsTpye =null; protected static final Logger LOG = LoggerFactory.getLogger(Task.class); private static Set keywordsFromString2HashSet(final String keywords) { return new HashSet(Arrays.asList(keywords.split(","))); } - public static void transferDatabaseKeywords(List keywords) { + public static void escapeDatabaseKeywords(List keywords) { //判断是否需要更改关键字集合 - if (databaseKeywords == null || currentKeywordsTpye != compatibleMode) { + if (databaseKeywords == null) { if (isOracleMode()) { databaseKeywords = keywordsFromString2HashSet(ORACLE_KEYWORDS); } else { databaseKeywords = keywordsFromString2HashSet(MYSQL_KEYWORDS); } - currentKeywordsTpye = compatibleMode; } char escapeChar = isOracleMode() ? '"' : '`'; for (int i = 0; i < keywords.size(); i++) { - String keyword = keywords.get(i).toUpperCase(); - if (databaseKeywords.contains(keyword)) { + String keyword = keywords.get(i); + if (databaseKeywords.contains(keyword.toUpperCase())) { keyword = escapeChar + keyword + escapeChar; } keyword = keyword.toLowerCase(); From 9f09039a23aee7fa42a4cd140c81aee7687518ce Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Mon, 27 Dec 2021 17:49:49 +0800 Subject: [PATCH 081/114] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/ext/ReaderTask.java | 26 ++++---- .../util/ObReaderUtils.java | 2 +- .../OceanBaseV10Writer.java | 30 +++++----- .../task/ConcurrentTableWriterTask.java | 59 +++++++++---------- .../util/ObWriterUtils.java | 1 - 5 files changed, 55 insertions(+), 63 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java index 254b334..1376f11 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java @@ -1,13 +1,5 @@ package com.alibaba.datax.plugin.reader.oceanbasev10reader.ext; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.plugin.RecordSender; @@ -19,11 +11,17 @@ import com.alibaba.datax.plugin.rdbms.reader.CommonRdbmsReader; import com.alibaba.datax.plugin.rdbms.reader.Constant; import com.alibaba.datax.plugin.rdbms.reader.Key; import com.alibaba.datax.plugin.rdbms.util.DBUtil; -import com.alibaba.datax.plugin.rdbms.util.DataBaseType; import com.alibaba.datax.plugin.rdbms.util.RdbmsException; import com.alibaba.datax.plugin.reader.oceanbasev10reader.Config; import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.ObReaderUtils; import com.alibaba.datax.plugin.reader.oceanbasev10reader.util.TaskContext; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; public class ReaderTask extends CommonRdbmsReader.Task { private static final Logger LOG = LoggerFactory.getLogger(ReaderTask.class); @@ -64,13 +62,9 @@ public class ReaderTask extends CommonRdbmsReader.Task { } } - if (ObReaderUtils.databaseType == DataBaseType.OceanBase) { - jdbcUrl = jdbcUrl.replace("jdbc:mysql:", "jdbc:oceanbase:") + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 - } else { - jdbcUrl = jdbcUrl + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 - } - if(ObReaderUtils.compatibleMode==ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE){ - compatibleMode=ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; + jdbcUrl = jdbcUrl.replace("jdbc:mysql:", "jdbc:oceanbase:") + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 + if (ObReaderUtils.compatibleMode == ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE) { + compatibleMode = ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; } LOG.info("this is ob1_0 jdbc url. user=" + username + " :url=" + jdbcUrl); mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java index 143171d..878aaea 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java @@ -717,6 +717,6 @@ public class ObReaderUtils { } public static boolean isOracleMode(String mode) { - return (mode != null && OB_COMPATIBLE_MODE_ORACLE.equals(mode.toString().toUpperCase())); + return (mode != null && OB_COMPATIBLE_MODE_ORACLE.equalsIgnoreCase(mode)); } } diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java index 417fccb..ede2eb0 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/OceanBaseV10Writer.java @@ -1,15 +1,5 @@ package com.alibaba.datax.plugin.writer.oceanbasev10writer; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; - -import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.DbUtils; -import com.alibaba.fastjson.JSONObject; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; @@ -20,7 +10,16 @@ import com.alibaba.datax.plugin.rdbms.writer.Constant; import com.alibaba.datax.plugin.rdbms.writer.Key; import com.alibaba.datax.plugin.rdbms.writer.util.WriterUtil; import com.alibaba.datax.plugin.writer.oceanbasev10writer.task.ConcurrentTableWriterTask; +import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.DbUtils; import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.ObWriterUtils; +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; /** * 2016-04-07 @@ -61,17 +60,17 @@ public class OceanBaseV10Writer extends Writer { this.originalConfig = super.getPluginJobConf(); checkCompatibleMode(originalConfig); //将config中的column和table中的关键字进行转义 - List columns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.writer.Key.COLUMN, String.class); + List columns = originalConfig.getList(Key.COLUMN, String.class); ObWriterUtils.escapeDatabaseKeywords(columns); - originalConfig.set(com.alibaba.datax.plugin.rdbms.writer.Key.COLUMN, columns); + originalConfig.set(Key.COLUMN, columns); - List conns = originalConfig.getList(com.alibaba.datax.plugin.rdbms.writer.Constant.CONN_MARK, JSONObject.class); + List conns = originalConfig.getList(Constant.CONN_MARK, JSONObject.class); for (int i = 0; i < conns.size(); i++) { JSONObject conn = conns.get(i); Configuration connConfig = Configuration.from(conn.toString()); - List tables = connConfig.getList(com.alibaba.datax.plugin.rdbms.writer.Key.TABLE, String.class); + List tables = connConfig.getList(Key.TABLE, String.class); ObWriterUtils.escapeDatabaseKeywords(tables); - originalConfig.set(String.format("%s[%d].%s", com.alibaba.datax.plugin.rdbms.writer.Constant.CONN_MARK, i, com.alibaba.datax.plugin.rdbms.writer.Key.TABLE), tables); + originalConfig.set(String.format("%s[%d].%s", Constant.CONN_MARK, i, Key.TABLE), tables); } this.commonJob = new CommonRdbmsWriter.Job(DATABASE_TYPE); this.commonJob.init(this.originalConfig); @@ -235,6 +234,7 @@ public class OceanBaseV10Writer extends Writer { /** * 注意：此方法每个 Task 都会执行一次。 最佳实践：此处适当封装确保简洁清晰完成数据写入工作。 */ + @Override public void startWrite(RecordReceiver recordReceiver) { this.writerTask.startWrite(recordReceiver, this.writerSliceConfig, super.getTaskPluginCollector()); } diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index f07c543..cf39cf2 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -1,7 +1,27 @@ package com.alibaba.datax.plugin.writer.oceanbasev10writer.task; +import com.alibaba.datax.common.element.Column; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; +import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.rdbms.util.DBUtil; +import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode; +import com.alibaba.datax.plugin.rdbms.util.DataBaseType; +import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter; +import com.alibaba.datax.plugin.writer.oceanbasev10writer.Config; +import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ConnHolder; +import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ObClientConnHolder; +import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ServerConnectInfo; +import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.ObWriterUtils; +import com.alipay.oceanbase.obproxy.data.TableEntryKey; +import com.alipay.oceanbase.obproxy.util.ObPartitionIdCalculator; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.sql.Connection; -//import java.sql.PreparedStatement; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; @@ -15,29 +35,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Pattern; -import com.alibaba.datax.common.element.Column; -import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ObClientConnHolder; -import org.apache.commons.lang3.tuple.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.alibaba.datax.common.element.Record; -import com.alibaba.datax.common.exception.DataXException; -import com.alibaba.datax.common.plugin.RecordReceiver; -import com.alibaba.datax.common.plugin.TaskPluginCollector; -import com.alibaba.datax.common.util.Configuration; -import com.alibaba.datax.plugin.rdbms.util.DBUtil; -import com.alibaba.datax.plugin.rdbms.util.DBUtilErrorCode; -import com.alibaba.datax.plugin.rdbms.util.DataBaseType; -import com.alibaba.datax.plugin.rdbms.writer.CommonRdbmsWriter; -import com.alibaba.datax.plugin.writer.oceanbasev10writer.Config; -import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ConnHolder; -import com.alibaba.datax.plugin.writer.oceanbasev10writer.ext.ServerConnectInfo; -import com.alibaba.datax.plugin.writer.oceanbasev10writer.util.ObWriterUtils; -import com.alipay.oceanbase.obproxy.data.TableEntryKey; -import com.alipay.oceanbase.obproxy.util.ObPartitionIdCalculator; +//import java.sql.PreparedStatement; public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { private static final Logger LOG = LoggerFactory.getLogger(ConcurrentTableWriterTask.class); @@ -105,14 +104,14 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { connectInfo.getFullUserName(), connectInfo.password); checkConnHolder.initConnection(); if (isOracleCompatibleMode) { - connectInfo.databaseName = connectInfo.databaseName.toUpperCase(); - //在转义的情况下不翻译 - if(!Pattern.matches("\"\\w*\"",table)){ - table = table.toUpperCase(); - } + connectInfo.databaseName = connectInfo.databaseName.toUpperCase(); + //在转义的情况下不翻译 + if (table.startsWith("\"") && table.endsWith("\"")) { + table = table.toUpperCase(); + } - LOG.info(String.format("this is oracle compatible mode, change database to %s, table to %s", - connectInfo.databaseName, table)); + LOG.info(String.format("this is oracle compatible mode, change database to %s, table to %s", + connectInfo.databaseName, table)); } if (config.getBool(Config.USE_PART_CALCULATOR, Config.DEFAULT_USE_PART_CALCULATOR)) { diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java index 8e9b420..072a0f2 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java @@ -41,7 +41,6 @@ public class ObWriterUtils { if (databaseKeywords.contains(keyword.toUpperCase())) { keyword = escapeChar + keyword + escapeChar; } - keyword = keyword.toLowerCase(); keywords.set(i, keyword); } } From b1334308a08b7c6ffdd2edbdc43f055ad1aef0ea Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Tue, 28 Dec 2021 10:10:43 +0800 Subject: [PATCH 082/114] =?UTF-8?q?=E6=8A=8A=E8=BD=AC=E4=B9=89=E7=9A=84?= =?UTF-8?q?=E9=80=BB=E8=BE=91=E9=87=8D=E6=96=B0=E6=8F=90=E4=BA=A4=E4=B8=80?= =?UTF-8?q?=E4=B8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10writer/task/ConcurrentTableWriterTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index cf39cf2..1e6d27c 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -106,7 +106,7 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { if (isOracleCompatibleMode) { connectInfo.databaseName = connectInfo.databaseName.toUpperCase(); //在转义的情况下不翻译 - if (table.startsWith("\"") && table.endsWith("\"")) { + if (!(table.startsWith("\"") && table.endsWith("\""))) { table = table.toUpperCase(); } From ce29ae7ee5d7ac5163c30b80ef0b41c361ced7b8 Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Tue, 28 Dec 2021 18:25:16 +0800 Subject: [PATCH 083/114] =?UTF-8?q?=E6=9C=AC=E6=AC=A1=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E8=BD=AC=E4=B9=89=E7=9A=84=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E4=B8=8D=E9=9C=80=E8=A6=81=E8=BF=9B=E8=A1=8C=E5=A4=A7?= =?UTF-8?q?=E5=B0=8F=E5=86=99=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../util/ObReaderUtils.java | 71 +++++++++---------- .../task/ConcurrentTableWriterTask.java | 38 +++++++--- .../util/ObWriterUtils.java | 44 +++++++++--- 3 files changed, 96 insertions(+), 57 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java index 878aaea..20c2f92 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/util/ObReaderUtils.java @@ -1,37 +1,22 @@ package com.alibaba.datax.plugin.reader.oceanbasev10reader.util; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.sql.Timestamp; -import java.util.*; -import java.util.Map.Entry; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - +import com.alibaba.datax.common.element.*; +import com.alibaba.datax.plugin.rdbms.util.DBUtil; import com.alibaba.datax.plugin.rdbms.util.DataBaseType; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLExpr; +import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; +import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.alibaba.datax.common.element.BoolColumn; -import com.alibaba.datax.common.element.BytesColumn; -import com.alibaba.datax.common.element.Column; -import com.alibaba.datax.common.element.DateColumn; -import com.alibaba.datax.common.element.DoubleColumn; -import com.alibaba.datax.common.element.LongColumn; -import com.alibaba.datax.common.element.Record; -import com.alibaba.datax.common.element.StringColumn; -import com.alibaba.datax.plugin.rdbms.util.DBUtil; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLExpr; -import com.alibaba.druid.sql.ast.expr.SQLBinaryOpExpr; -import com.alibaba.druid.sql.ast.expr.SQLBinaryOperator; - -import javax.xml.crypto.Data; +import java.sql.*; +import java.util.*; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class ObReaderUtils { @@ -53,21 +38,32 @@ public class ObReaderUtils { return new HashSet(Arrays.asList(keywords.split(","))); } - public static void escapeDatabaseKeywords(List keywords) { + public static String escapeDatabaseKeywords(String keyword) { if (databaseKeywords == null) { - if (isOracleMode(compatibleMode.toString())) { + if (isOracleMode(compatibleMode)) { databaseKeywords = keywordsFromString2HashSet(ORACLE_KEYWORDS); } else { databaseKeywords = keywordsFromString2HashSet(MYSQL_KEYWORDS); } } - char escapeChar = isOracleMode(compatibleMode.toString()) ? '"' : '`'; + char escapeChar = isOracleMode(compatibleMode) ? '"' : '`'; + if (databaseKeywords.contains(keyword.toUpperCase())) { + keyword = escapeChar + keyword + escapeChar; + } + return keyword; + } + + public static void escapeDatabaseKeywords(List keywords) { for (int i = 0; i < keywords.size(); i++) { - String keyword = keywords.get(i); - if (databaseKeywords.contains(keyword.toUpperCase())) { - keyword = escapeChar + keyword + escapeChar; - } - keywords.set(i, keyword); + keywords.set(i, escapeDatabaseKeywords(keywords.get(i))); + } + } + + public static Boolean isEscapeMode(String keyword) { + if (isOracleMode(compatibleMode)) { + return keyword.startsWith("\"") && keyword.endsWith("\""); + } else { + return keyword.startsWith("`") && keyword.endsWith("`"); } } @@ -151,12 +147,15 @@ public class ObReaderUtils { ps = conn.createStatement(); rs = ps.executeQuery(sql); while (rs.next()) { - String columnName = StringUtils.lowerCase(rs.getString("Column_name")); + String columnName = rs.getString("Column_name"); + columnName = escapeDatabaseKeywords(columnName); + if (!isEscapeMode(columnName)) { + columnName.toLowerCase(); + } if (!realIndex.contains(columnName)) { realIndex.add(columnName); } } - escapeDatabaseKeywords(realIndex); String[] pks = new String[realIndex.size()]; realIndex.toArray(pks); diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index 1e6d27c..eb1222c 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -62,6 +62,7 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { private ObPartitionIdCalculator partCalculator = null; private HashMap> groupInsertValues; + List unknownPartRecords = new ArrayList(); // private List unknownPartRecords; private List partitionKeyIndexes; @@ -307,6 +308,18 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { } } } + if(unknownPartRecords.size()>0){ + int retry = 0; + while (true) { + try { + concurrentWriter.addBatchRecords(unknownPartRecords); + break; + } catch (InterruptedException e) { + retry++; + LOG.info("Concurrent table writer is interrupted, retry {}", retry); + } + } + } } private void addRecordToCache(final Record record) { @@ -347,21 +360,24 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { groupInsertValues.put(partId, groupValues); } } else { - LOG.warn("add unknown part record {}", record); - List unknownPartRecords = new ArrayList(); + LOG.debug("add unknown part record {}", record); + unknownPartRecords.add(record); int i = 0; - while (true) { - if (i > 0) { - LOG.info("retry add batch record the {} times", i); - } - try { - concurrentWriter.addBatchRecords(unknownPartRecords); - break; - } catch (InterruptedException e) { - LOG.info("Concurrent table writer is interrupted"); + if(unknownPartRecords.size()>batchSize){ + while (true) { + if (i > 0) { + LOG.info("retry add batch record the {} times", i); + } + try { + concurrentWriter.addBatchRecords(unknownPartRecords); + break; + } catch (InterruptedException e) { + LOG.info("Concurrent table writer is interrupted"); + } } } + } } diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java index 072a0f2..f4c4f43 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java @@ -11,7 +11,6 @@ import org.slf4j.LoggerFactory; import java.sql.*; import java.util.*; -import java.util.stream.Collectors; public class ObWriterUtils { @@ -26,8 +25,8 @@ public class ObWriterUtils { return new HashSet(Arrays.asList(keywords.split(","))); } - public static void escapeDatabaseKeywords(List keywords) { - //判断是否需要更改关键字集合 + //java中的String的坑 + public static String escapeDatabaseKeywords(String keyword) { if (databaseKeywords == null) { if (isOracleMode()) { databaseKeywords = keywordsFromString2HashSet(ORACLE_KEYWORDS); @@ -36,12 +35,22 @@ public class ObWriterUtils { } } char escapeChar = isOracleMode() ? '"' : '`'; + if (databaseKeywords.contains(keyword.toUpperCase())) { + keyword = escapeChar + keyword + escapeChar; + } + return keyword; + } + + public static void escapeDatabaseKeywords(List keywords) { for (int i = 0; i < keywords.size(); i++) { - String keyword = keywords.get(i); - if (databaseKeywords.contains(keyword.toUpperCase())) { - keyword = escapeChar + keyword + escapeChar; - } - keywords.set(i, keyword); + keywords.set(i, escapeDatabaseKeywords(keywords.get(i))); + } + } + public static Boolean isEscapeMode(String keyword){ + if(isOracleMode()){ + return keyword.startsWith("\"") && keyword.endsWith("\""); + }else{ + return keyword.startsWith("`") && keyword.endsWith("`"); } } public static boolean isMemstoreFull(Connection conn, double memstoreThreshold) { @@ -94,7 +103,16 @@ public class ObWriterUtils { } private static int[] getColumnIndex(List columnsInIndex, List allColumns) { - allColumns = allColumns.stream().map(String::toUpperCase).collect(Collectors.toList()); + /** + * JDK8的stream模型：将一种数据结构转化成通用的数据模型，并可在该模型上进行操作 + * map:接受一个函数引用，用于操作元素 + * collect：接受一个Collectors方法，用于将中间数据模型转化成目标数据结构 + */ + for (int i = 0; i < allColumns.size(); i++) { + if (!ObWriterUtils.isEscapeMode(allColumns.get(i))) { + allColumns.set(i, allColumns.get(i).toUpperCase()); + } + } int[] colIdx = new int[columnsInIndex.size()]; for (int i = 0; i < columnsInIndex.size(); i++) { int index = allColumns.indexOf(columnsInIndex.get(i)); @@ -146,7 +164,11 @@ public class ObWriterUtils { rs = stmt.executeQuery(sql); while (rs.next()) { String keyName = rs.getString("Key_name"); - String columnName = StringUtils.upperCase(rs.getString("Column_name")); + String columnName = rs.getString("Column_name"); + columnName=escapeDatabaseKeywords(columnName); + if(!ObWriterUtils.isEscapeMode(columnName)){ + columnName=columnName.toUpperCase(); + } List s = uniqueKeys.get(keyName); if (s == null) { s = new ArrayList(); @@ -159,6 +181,7 @@ public class ObWriterUtils { } finally { asyncClose(rs, stmt, null); } + //ObWriterUtils.escapeDatabaseKeywords(uniqueKeys); return uniqueKeys; } @@ -315,6 +338,7 @@ public class ObWriterUtils { * @param e * @return */ + public static boolean isFatalError(SQLException e) { String sqlState = e.getSQLState(); if (StringUtils.startsWith(sqlState, "08")) { From 042aa2d86599162367024a3072fdb34547736a4e Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Tue, 28 Dec 2021 19:32:15 +0800 Subject: [PATCH 084/114] =?UTF-8?q?=E5=9C=A8=E4=B8=8A=E4=B8=AA=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E7=9A=84=E5=9F=BA=E7=A1=80=E4=B8=8A=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96=E4=BA=86=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10writer/task/ConcurrentTableWriterTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index eb1222c..bed0598 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -308,7 +308,7 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { } } } - if(unknownPartRecords.size()>0){ + if (unknownPartRecords.size() > 0) { int retry = 0; while (true) { try { @@ -364,7 +364,7 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { unknownPartRecords.add(record); int i = 0; - if(unknownPartRecords.size()>batchSize){ + if (unknownPartRecords.size() > batchSize) { while (true) { if (i > 0) { LOG.info("retry add batch record the {} times", i); From c96a366a4eab44acc114f7a31275ba243701352c Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Wed, 29 Dec 2021 19:23:34 +0800 Subject: [PATCH 085/114] =?UTF-8?q?=E6=8A=8A=E5=86=97=E4=BD=99=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=90=88=E5=B9=B6=E6=88=90=E4=BA=86=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E5=87=BD=E6=95=B0=EF=BC=8C=E5=88=A0=E9=99=A4=E4=BA=86=E4=B8=8D?= =?UTF-8?q?=E5=BF=85=E8=A6=81=E7=9A=84=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10reader/ext/ReaderTask.java | 2 +- .../task/ConcurrentTableWriterTask.java | 72 +++++++------------ .../util/ObWriterUtils.java | 8 +-- 3 files changed, 28 insertions(+), 54 deletions(-) diff --git a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java index 1376f11..a43dceb 100644 --- a/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java +++ b/oceanbasev10reader/src/main/java/com/alibaba/datax/plugin/reader/oceanbasev10reader/ext/ReaderTask.java @@ -63,7 +63,7 @@ public class ReaderTask extends CommonRdbmsReader.Task { } jdbcUrl = jdbcUrl.replace("jdbc:mysql:", "jdbc:oceanbase:") + "&socketTimeout=1800000&connectTimeout=60000"; //socketTimeout 半个小时 - if (ObReaderUtils.compatibleMode == ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE) { + if (ObReaderUtils.compatibleMode.equals(ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE)) { compatibleMode = ObReaderUtils.OB_COMPATIBLE_MODE_ORACLE; } LOG.info("this is ob1_0 jdbc url. user=" + username + " :url=" + jdbcUrl); diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index bed0598..bd34b4a 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -294,31 +294,14 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { } private void addLeftRecords() { + //不需要刷新Cache，已经是最后一批数据了 for (List groupValues : groupInsertValues.values()) { if (groupValues.size() > 0 ) { - int retry = 0; - while (true) { - try { - concurrentWriter.addBatchRecords(groupValues); - break; - } catch (InterruptedException e) { - retry++; - LOG.info("Concurrent table writer is interrupted, retry {}", retry); - } - } + addRecordsToWriteQueue(groupValues); } } if (unknownPartRecords.size() > 0) { - int retry = 0; - while (true) { - try { - concurrentWriter.addBatchRecords(unknownPartRecords); - break; - } catch (InterruptedException e) { - retry++; - LOG.info("Concurrent table writer is interrupted, retry {}", retry); - } - } + addRecordsToWriteQueue(unknownPartRecords); } } @@ -344,43 +327,40 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { groupValues.add(record); if (groupValues.size() >= batchSize) { int i = 0; - while (true) { - if (i > 0) { - LOG.info("retry add batch record the {} times", i); - } - try { - concurrentWriter.addBatchRecords(groupValues); - printEveryTime(); - break; - } catch (InterruptedException e) { - LOG.info("Concurrent table writer is interrupted"); - } - } - groupValues = new ArrayList(batchSize); + groupValues =addRecordsToWriteQueue(groupValues); groupInsertValues.put(partId, groupValues); } } else { LOG.debug("add unknown part record {}", record); - unknownPartRecords.add(record); - int i = 0; if (unknownPartRecords.size() > batchSize) { - while (true) { - if (i > 0) { - LOG.info("retry add batch record the {} times", i); - } - try { - concurrentWriter.addBatchRecords(unknownPartRecords); - break; - } catch (InterruptedException e) { - LOG.info("Concurrent table writer is interrupted"); - } - } + unknownPartRecords=addRecordsToWriteQueue(unknownPartRecords); } } } + /** + * + * @param records + * @return 返回一个新的Cache用于存储接下来的数据 + */ + private List addRecordsToWriteQueue(List records) { + int i = 0; + while (true) { + if (i > 0) { + LOG.info("retry add batch record the {} times", i); + } + try { + concurrentWriter.addBatchRecords(records); + break; + } catch (InterruptedException e) { + i++; + LOG.info("Concurrent table writer is interrupted"); + } + } + return new ArrayList(batchSize); + } private void checkMemStore() { Connection checkConn = checkConnHolder.reconnect(); long now = System.currentTimeMillis(); diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java index f4c4f43..ff1648a 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/util/ObWriterUtils.java @@ -25,7 +25,6 @@ public class ObWriterUtils { return new HashSet(Arrays.asList(keywords.split(","))); } - //java中的String的坑 public static String escapeDatabaseKeywords(String keyword) { if (databaseKeywords == null) { if (isOracleMode()) { @@ -103,11 +102,6 @@ public class ObWriterUtils { } private static int[] getColumnIndex(List columnsInIndex, List allColumns) { - /** - * JDK8的stream模型：将一种数据结构转化成通用的数据模型，并可在该模型上进行操作 - * map:接受一个函数引用，用于操作元素 - * collect：接受一个Collectors方法，用于将中间数据模型转化成目标数据结构 - */ for (int i = 0; i < allColumns.size(); i++) { if (!ObWriterUtils.isEscapeMode(allColumns.get(i))) { allColumns.set(i, allColumns.get(i).toUpperCase()); @@ -167,7 +161,7 @@ public class ObWriterUtils { String columnName = rs.getString("Column_name"); columnName=escapeDatabaseKeywords(columnName); if(!ObWriterUtils.isEscapeMode(columnName)){ - columnName=columnName.toUpperCase(); + columnName = columnName.toUpperCase(); } List s = uniqueKeys.get(keyName); if (s == null) { From c140ca35fba4f01e71bd492a2f5376e4d0877383 Mon Sep 17 00:00:00 2001 From: sanChouIsACat <993924507@qq.com> Date: Thu, 30 Dec 2021 11:09:58 +0800 Subject: [PATCH 086/114] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BA=86=E4=B8=8A?= =?UTF-8?q?=E8=BF=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oceanbasev10writer/task/ConcurrentTableWriterTask.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java index bd34b4a..e6b4a56 100644 --- a/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java +++ b/oceanbasev10writer/src/main/java/com/alibaba/datax/plugin/writer/oceanbasev10writer/task/ConcurrentTableWriterTask.java @@ -326,15 +326,14 @@ public class ConcurrentTableWriterTask extends CommonRdbmsWriter.Task { } groupValues.add(record); if (groupValues.size() >= batchSize) { - int i = 0; - groupValues =addRecordsToWriteQueue(groupValues); + groupValues = addRecordsToWriteQueue(groupValues); groupInsertValues.put(partId, groupValues); } } else { LOG.debug("add unknown part record {}", record); unknownPartRecords.add(record); - if (unknownPartRecords.size() > batchSize) { - unknownPartRecords=addRecordsToWriteQueue(unknownPartRecords); + if (unknownPartRecords.size() >= batchSize) { + unknownPartRecords = addRecordsToWriteQueue(unknownPartRecords); } } From b71cdf693e418a8674f39b05d38ee2f350f0791f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A1=BE=E8=BD=A9?= Date: Thu, 13 Jan 2022 17:53:30 +0800 Subject: [PATCH 087/114] fix compile problem --- oscarwriter/pom.xml | 8 +++- .../tsdbreader/conn/TSDBConnectionTest.java | 4 +- .../reader/tsdbreader/util/HttpUtilsTest.java | 39 ------------------- .../writer/conn/TSDBConnectionTest.java | 4 +- .../plugin/writer/util/HttpUtilsTest.java | 4 +- .../datax/plugin/writer/util/TSDBTest.java | 4 +- 6 files changed, 15 insertions(+), 48 deletions(-) delete mode 100644 tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java diff --git a/oscarwriter/pom.xml b/oscarwriter/pom.xml index 51643c7..06249a2 100644 --- a/oscarwriter/pom.xml +++ b/oscarwriter/pom.xml @@ -39,12 +39,18 @@ plugin-rdbms-util ${datax-project-version} - + + + + com.csicit.thirdparty + oscar + 1.0.1 diff --git a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java index e454408..6be291e 100644 --- a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java +++ b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java @@ -19,12 +19,12 @@ public class TSDBConnectionTest { @Test public void testVersion() { - String version = new TSDBConnection(TSDB_ADDRESS).version(); + String version = new TSDBConnection(TSDB_ADDRESS,null,null).version(); Assert.assertNotNull(version); } @Test public void testIsSupported() { - Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS).isSupported()); + Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS,null,null).isSupported()); } } diff --git a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java b/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java deleted file mode 100644 index 12a2660..0000000 --- a/tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.alibaba.datax.plugin.reader.tsdbreader.util; - -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -/** - * Copyright @ 2019 alibaba.com - * All right reserved. - * Function：HttpUtils Test - * - * @author Benedict Jin - * @since 2019-10-21 - */ -@Ignore -public class HttpUtilsTest { - - @Test - public void testSimpleCase() throws Exception { - String url = "https://httpbin.org/post"; - Map params = new HashMap<>(); - params.put("foo", "bar"); - - String rsp = HttpUtils.post(url, params); - System.out.println(rsp); - Assert.assertNotNull(rsp); - } - - @Test - public void testGet() throws Exception { - String url = String.format("%s/api/version", Const.TSDB_ADDRESS); - String rsp = HttpUtils.get(url); - System.out.println(rsp); - Assert.assertNotNull(rsp); - } -} diff --git a/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/conn/TSDBConnectionTest.java b/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/conn/TSDBConnectionTest.java index 455f4ce..fada706 100644 --- a/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/conn/TSDBConnectionTest.java +++ b/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/conn/TSDBConnectionTest.java @@ -19,12 +19,12 @@ public class TSDBConnectionTest { @Test public void testVersion() { - String version = new TSDBConnection(TSDB_ADDRESS).version(); + String version = new TSDBConnection(TSDB_ADDRESS,null,null,null).version(); Assert.assertNotNull(version); } @Test public void testIsSupported() { - Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS).isSupported()); + Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS,null,null,null).isSupported()); } } diff --git a/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/HttpUtilsTest.java b/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/HttpUtilsTest.java index 69f26b8..1f8fb87 100644 --- a/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/HttpUtilsTest.java +++ b/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/HttpUtilsTest.java @@ -24,7 +24,7 @@ public class HttpUtilsTest { Map params = new HashMap(); params.put("foo", "bar"); - String rsp = HttpUtils.post(url, params); + String rsp = HttpUtils.post(url, null,null,params); System.out.println(rsp); Assert.assertNotNull(rsp); } @@ -32,7 +32,7 @@ public class HttpUtilsTest { @Test public void testGet() throws Exception { String url = String.format("%s/api/version", Const.OPENTSDB_ADDRESS); - String rsp = HttpUtils.get(url); + String rsp = HttpUtils.get(url,null,null); System.out.println(rsp); Assert.assertNotNull(rsp); } diff --git a/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/TSDBTest.java b/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/TSDBTest.java index 7d22bb7..8debf40 100644 --- a/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/TSDBTest.java +++ b/tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/TSDBTest.java @@ -17,11 +17,11 @@ public class TSDBTest { @Test public void testVersion() { - String version = TSDBUtils.version(Const.TSDB_ADDRESS); + String version = TSDBUtils.version(Const.TSDB_ADDRESS,null,null); Assert.assertNotNull(version); System.out.println(version); - version = TSDBUtils.version(Const.OPENTSDB_ADDRESS); + version = TSDBUtils.version(Const.OPENTSDB_ADDRESS,null,null); Assert.assertNotNull(version); System.out.println(version); } From 155e93b9d89f10971da4d0c177077c7536f71135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A1=BE=E8=BD=A9?= Date: Thu, 13 Jan 2022 19:36:22 +0800 Subject: [PATCH 088/114] fix log4j --- otsreader/pom.xml | 10 ++++++++++ otsstreamreader/pom.xml | 10 ++++++++++ otswriter/pom.xml | 10 ++++++++++ pom.xml | 11 +++++++++++ 4 files changed, 41 insertions(+) diff --git a/otsreader/pom.xml b/otsreader/pom.xml index bd01742..b1e0e73 100644 --- a/otsreader/pom.xml +++ b/otsreader/pom.xml @@ -34,6 +34,16 @@ com.aliyun.openservices ots-public 2.2.4 + + + log4j-api + org.apache.logging.log4j + + + log4j-core + org.apache.logging.log4j + + com.google.code.gson diff --git a/otsstreamreader/pom.xml b/otsstreamreader/pom.xml index 2a12872..84ca2d6 100644 --- a/otsstreamreader/pom.xml +++ b/otsstreamreader/pom.xml @@ -33,6 +33,16 @@ com.aliyun.openservices tablestore-streamclient 1.0.0 + + + log4j-api + org.apache.logging.log4j + + + log4j-core + org.apache.logging.log4j + + com.google.code.gson diff --git a/otswriter/pom.xml b/otswriter/pom.xml index 8677c8a..d40f68b 100644 --- a/otswriter/pom.xml +++ b/otswriter/pom.xml @@ -34,6 +34,16 @@ com.aliyun.openservices ots-public 2.2.4 + + + log4j-api + org.apache.logging.log4j + + + log4j-core + org.apache.logging.log4j + + com.google.code.gson diff --git a/pom.xml b/pom.xml index 1a9da81..7273d60 100644 --- a/pom.xml +++ b/pom.xml @@ -183,6 +183,17 @@ 1.9.5 test + + org.apache.logging.log4j + log4j-api + 2.17.0 + + + + org.apache.logging.log4j + log4j-core + 2.17.0 + From c655685acd8dcc5edfb496a99cc51d9096b3f5a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 11:37:37 +0000 Subject: [PATCH 089/114] Bump log4j-core from 2.17.0 to 2.17.1 Bumps log4j-core from 2.17.0 to 2.17.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7273d60..9921d2f 100644 --- a/pom.xml +++ b/pom.xml @@ -192,7 +192,7 @@ org.apache.logging.log4j log4j-core - 2.17.0 + 2.17.1 From 22bf5b537b3d2f23b7c495d01ff6e02d8d3e9c66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 11:37:38 +0000 Subject: [PATCH 090/114] Bump log4j-api from 2.17.0 to 2.17.1 Bumps log4j-api from 2.17.0 to 2.17.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-api dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7273d60..1d211b4 100644 --- a/pom.xml +++ b/pom.xml @@ -186,7 +186,7 @@ org.apache.logging.log4j log4j-api - 2.17.0 + 2.17.1 From df3ea169b018b9cf03e6e32fdbd94c42b4b115c6 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 14 Feb 2022 18:58:17 +0800 Subject: [PATCH 091/114] tdenginewriter rebuiled --- tdenginereader/pom.xml | 2 +- .../com/alibaba/datax/plugin/reader/Key.java | 2 +- .../datax/plugin/reader/TDengineReader.java | 5 +- tdenginewriter/pom.xml | 28 +- .../writer/tdenginewriter/ColumnMeta.java | 24 + .../writer/tdenginewriter/Constants.java | 8 + .../writer/tdenginewriter/DataHandler.java | 5 +- .../tdenginewriter/DataHandlerFactory.java | 10 - .../tdenginewriter/DefaultDataHandler.java | 473 +++++++++++++++--- .../tdenginewriter/JDBCBatchWriter.java | 244 --------- .../writer/tdenginewriter/JniConnection.java | 89 ---- .../plugin/writer/tdenginewriter/Key.java | 16 +- .../tdenginewriter/OpentsdbDataHandler.java | 68 +-- .../writer/tdenginewriter/SchemaManager.java | 349 +++++-------- .../writer/tdenginewriter/TDengineWriter.java | 78 +-- .../TDengineWriterErrorCode.java | 12 +- .../writer/tdenginewriter/TableMeta.java | 22 + .../writer/tdenginewriter/TableType.java | 5 + .../tdenginewriter/TimestampPrecision.java | 5 + ...ugin_writer_tdenginewriter_JniConnection.h | 105 ---- .../DefaultDataHandlerTest.java | 235 +++++++++ .../DefaultDataHandlerTest2.java | 85 ++++ .../writer/tdenginewriter/EngineTest.java | 65 +++ .../tdenginewriter/JniConnectionTest.java | 21 - .../tdenginewriter/SchemaManagerTest.java | 88 ++++ .../tdenginewriter/TDengineWriterTest.java | 69 ++- .../src/test/resources/mysql2tdengine.json | 57 +++ .../src/test/resources/opentsdb2tdengine.json | 36 ++ .../src/test/resources/tdengine2tdengine.json | 51 ++ 29 files changed, 1374 insertions(+), 883 deletions(-) create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/ColumnMeta.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java delete mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java delete mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java delete mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableMeta.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableType.java create mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TimestampPrecision.java delete mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java create mode 100644 tdenginewriter/src/test/resources/mysql2tdengine.json create mode 100644 tdenginewriter/src/test/resources/opentsdb2tdengine.json create mode 100644 tdenginewriter/src/test/resources/tdengine2tdengine.json diff --git a/tdenginereader/pom.xml b/tdenginereader/pom.xml index 8c0f664..ca444bc 100644 --- a/tdenginereader/pom.xml +++ b/tdenginereader/pom.xml @@ -32,7 +32,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.34 + 2.0.37 diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java index eddf98e..95b5538 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java @@ -7,7 +7,7 @@ public class Key { // public static final String PORT = "port"; // public static final String DB = "db"; public static final String TABLE = "table"; - public static final String USER = "user"; + public static final String USER = "username"; public static final String PASSWORD = "password"; public static final String CONNECTION = "connection"; // public static final String SQL = "sql"; diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index dfdce7b..f00d879 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -190,8 +190,9 @@ public class TDengineReader extends Reader { @Override public void startRead(RecordSender recordSender) { try (Statement stmt = conn.createStatement()) { - for (int i = 0; i < tables.size(); i++) { - String sql = "select " + StringUtils.join(columns, ",") + " from " + tables.get(i) + " where _c0 >= " + startTime + " and _c0 < " + endTime; + for (String table : tables) { + String sql = "select " + StringUtils.join(columns, ",") + " from " + table + + " where _c0 >= " + startTime + " and _c0 < " + endTime; ResultSet rs = stmt.executeQuery(sql); ResultSetMetaData metaData = rs.getMetaData(); while (rs.next()) { diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 054f2ef..720ecbb 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -19,11 +19,25 @@ + + + com.alibaba + fastjson + 1.2.58 + + com.taosdata.jdbc taos-jdbcdriver - 2.0.34 + 2.0.37 + + + com.alibaba + fastjson + + + com.alibaba.datax datax-common @@ -36,18 +50,18 @@ - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.34 - - junit junit ${junit-version} test + + com.alibaba.datax + datax-core + 0.0.1-SNAPSHOT + test + org.apache.commons commons-lang3 diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/ColumnMeta.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/ColumnMeta.java new file mode 100644 index 0000000..5c77ecc --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/ColumnMeta.java @@ -0,0 +1,24 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public class ColumnMeta { + String field; + String type; + int length; + String note; + boolean isTag; + boolean isPrimaryKey; + Object value; + + @Override + public String toString() { + return "ColumnMeta{" + + "field='" + field + '\'' + + ", type='" + type + '\'' + + ", length=" + length + + ", note='" + note + '\'' + + ", isTag=" + isTag + + ", isPrimaryKey=" + isPrimaryKey + + ", value=" + value + + '}'; + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java new file mode 100644 index 0000000..d62c8f3 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java @@ -0,0 +1,8 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public class Constants { + public static final String DEFAULT_USERNAME = "root"; + public static final String DEFAULT_PASSWORD = "taosdata"; + public static final int DEFAULT_BATCH_SIZE = 1; + public static final boolean DEFAULT_IGNORE_TAGS_UNMATCHED = false; +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java index dcc3ca8..7afcb08 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java @@ -1,12 +1,9 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.plugin.RecordReceiver; - import com.alibaba.datax.common.plugin.TaskPluginCollector; -import java.util.Properties; - public interface DataHandler { - long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector); + int handle(RecordReceiver lineReceiver, TaskPluginCollector collector); } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java deleted file mode 100644 index 1f740d7..0000000 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandlerFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -public class DataHandlerFactory { - - public static DataHandler build(String peerPluginName) { - if (peerPluginName.equals("opentsdbreader")) - return new OpentsdbDataHandler(); - return new DefaultDataHandler(); - } -} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index a8704f2..16be4c9 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -1,108 +1,423 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; +import com.alibaba.datax.common.element.Column; import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.plugin.TaskPluginCollector; -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.TSDBPreparedStatement; +import com.alibaba.datax.common.util.Configuration; +import com.taosdata.jdbc.SchemalessWriter; +import com.taosdata.jdbc.enums.SchemalessProtocolType; +import com.taosdata.jdbc.enums.SchemalessTimestampType; +import com.taosdata.jdbc.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Properties; +import java.sql.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; -/** - * 默认DataHandler - */ public class DefaultDataHandler implements DataHandler { private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); - static { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } + private String username; + private String password; + private String jdbcUrl; + private int batchSize; + private boolean ignoreTagsUnmatched; + + private List tables; + private List columns; + + private Map tableMetas; + private SchemaManager schemaManager; + + public void setTableMetas(Map tableMetas) { + this.tableMetas = tableMetas; + } + + public void setColumnMetas(Map> columnMetas) { + this.columnMetas = columnMetas; + } + + public void setSchemaManager(SchemaManager schemaManager) { + this.schemaManager = schemaManager; + } + + private Map> columnMetas; + + public DefaultDataHandler(Configuration configuration) { + this.username = configuration.getString(Key.USERNAME, Constants.DEFAULT_USERNAME); + this.password = configuration.getString(Key.PASSWORD, Constants.DEFAULT_PASSWORD); + this.jdbcUrl = configuration.getString(Key.JDBC_URL); + this.batchSize = configuration.getInt(Key.BATCH_SIZE, Constants.DEFAULT_BATCH_SIZE); + this.tables = configuration.getList(Key.TABLE, String.class); + this.columns = configuration.getList(Key.COLUMN, String.class); + this.ignoreTagsUnmatched = configuration.getBool(Key.IGNORE_TAGS_UNMATCHED, Constants.DEFAULT_IGNORE_TAGS_UNMATCHED); } @Override - public long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector) { - SchemaManager schemaManager = new SchemaManager(properties); - if (!schemaManager.configValid()) { - return 0; - } + public int handle(RecordReceiver lineReceiver, TaskPluginCollector collector) { + int count = 0; + int affectedRows = 0; - try { - Connection conn = getTaosConnection(properties); - if (conn == null) { - return 0; - } - if (schemaManager.shouldGuessSchema()) { - // 无法从配置文件获取表结构信息，尝试从数据库获取 - LOG.info(Msg.get("try_get_schema_from_db")); - boolean success = schemaManager.getFromDB(conn); - if (!success) { - return 0; + try (Connection conn = DriverManager.getConnection(jdbcUrl, username, password)) { + LOG.info("connection[ jdbcUrl: " + jdbcUrl + ", username: " + username + "] established."); + // prepare table_name -> table_meta + this.schemaManager = new SchemaManager(conn); + this.tableMetas = schemaManager.loadTableMeta(tables); + // prepare table_name -> column_meta + this.columnMetas = schemaManager.loadColumnMetas(tables); + + List recordBatch = new ArrayList<>(); + Record record; + for (int i = 1; (record = lineReceiver.getFromReader()) != null; i++) { + if (i % batchSize != 0) { + recordBatch.add(record); + } else { + affectedRows = writeBatch(conn, recordBatch); + recordBatch.clear(); } - } else { - + count++; } - int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, "1000")); - if (batchSize < 5) { - // batchSize太小，会增加自动类型推断错误的概率，建议改大后重试 - LOG.error(Msg.get("batch_size_too_small")); - return 0; - } - return write(lineReceiver, conn, batchSize, schemaManager, collector); - } catch (Exception e) { - LOG.error("write failed " + e.getMessage()); - e.printStackTrace(); + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); } - return 0; - } - - private Connection getTaosConnection(Properties properties) throws SQLException { - // 检查必要参数 - String host = properties.getProperty(Key.HOST); - String port = properties.getProperty(Key.PORT); - String dbname = properties.getProperty(Key.DBNAME); - String user = properties.getProperty(Key.USER); - String password = properties.getProperty(Key.PASSWORD); - if (host == null || port == null || dbname == null || user == null || password == null) { - String keys = String.join(" ", Key.HOST, Key.PORT, Key.DBNAME, Key.USER, Key.PASSWORD); - LOG.error("Required options missing, please check: " + keys); - return null; + if (affectedRows != count) { + LOG.error("write record missing or incorrect happened, affectedRows: " + affectedRows + ", total: " + count); } - String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", host, port, dbname, user, password); - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - LOG.info("TDengine connection established, host:{} port:{} dbname:{} user:{}", host, port, dbname, user); - return DriverManager.getConnection(jdbcUrl, connProps); + + return affectedRows; } /** - * 使用SQL批量写入
- * - * @return 成功写入记录数 - * @throws SQLException + * table: [ "stb1", "stb2", "tb1", "tb2", "t1" ] + * stb1[ts,f1,f2] tags:[t1] + * stb2[ts,f1,f2,f3] tags:[t1,t2] + * 1. tables 表的的类型分成：stb(super table)/tb(sub table)/t(original table) + * 2. 对于stb，自动建表/schemaless + * 2.1: data中有tbname字段, 例如：data: [ts, f1, f2, f3, t1, t2, tbname] tbColumn: [ts, f1, f2, t1] => insert into tbname using stb1 tags(t1) values(ts, f1, f2) + * 2.2: data中没有tbname字段，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, t1] => schemaless: stb1,t1=t1 f1=f1,f2=f2 ts, 没有批量写 + * 3. 对于tb，拼sql，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, t1] => insert into tb(ts, f1, f2) values(ts, f1, f2) + * 4. 对于t，拼sql，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, f3, t1, t2] insert into t(ts, f1, f2, f3, t1, t2) values(ts, f1, f2, f3, t1, t2) */ - private long write(RecordReceiver lineReceiver, Connection conn, int batchSize, SchemaManager scm, TaskPluginCollector collector) throws SQLException { - Record record = lineReceiver.getFromReader(); - if (record == null) { - return 0; + public int writeBatch(Connection conn, List recordBatch) { + int affectedRows = 0; + for (String table : tables) { + TableMeta tableMeta = tableMetas.get(table); + switch (tableMeta.tableType) { + case SUP_TABLE: { + if (columns.contains("tbname")) + affectedRows += writeBatchToSupTableBySQL(conn, table, recordBatch); + else + affectedRows += writeBatchToSupTableBySchemaless(conn, table, recordBatch); + } + break; + case SUB_TABLE: + affectedRows += writeBatchToSubTable(conn, table, recordBatch); + break; + case NML_TABLE: + default: + affectedRows += writeBatchToNormalTable(conn, table, recordBatch); + } } - String pq = String.format("INSERT INTO ? USING %s TAGS(%s) (%s) values (%s)", scm.getStable(), scm.getTagValuesPlaceHolder(), scm.getJoinedFieldNames(), scm.getFieldValuesPlaceHolder()); - LOG.info("Prepared SQL: {}", pq); - try (TSDBPreparedStatement stmt = (TSDBPreparedStatement) conn.prepareStatement(pq)) { - JDBCBatchWriter batchWriter = new JDBCBatchWriter(conn, stmt, scm, batchSize, collector); - do { - batchWriter.append(record); - } while ((record = lineReceiver.getFromReader()) != null); - batchWriter.flush(); - return batchWriter.getCount(); + return affectedRows; + } + + /** + * insert into record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) + * record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) + * record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) + */ + private int writeBatchToSupTableBySQL(Connection conn, String table, List recordBatch) { + List columnMetas = this.columnMetas.get(table); + + StringBuilder sb = new StringBuilder("insert into"); + for (Record record : recordBatch) { + sb.append(" ").append(record.getColumn(indexOf("tbname")).asString()) + .append(" using ").append(table) + .append(" tags") + .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return colMeta.isTag; + }).map(colMeta -> { + return buildColumnValue(colMeta, record); + }).collect(Collectors.joining(",", "(", ")"))) + .append(" ") + .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return !colMeta.isTag; + }).map(colMeta -> { + return colMeta.field; + }).collect(Collectors.joining(",", "(", ")"))) + .append(" values") + .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return !colMeta.isTag; + }).map(colMeta -> { + return buildColumnValue(colMeta, record); + }).collect(Collectors.joining(",", "(", ")"))); + } + String sql = sb.toString(); + + return executeUpdate(conn, sql); + } + + private int executeUpdate(Connection conn, String sql) throws DataXException { + int count; + try (Statement stmt = conn.createStatement()) { + LOG.debug(">>> " + sql); + count = stmt.executeUpdate(sql); + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + } + return count; + } + + private String buildColumnValue(ColumnMeta colMeta, Record record) { + Column column = record.getColumn(indexOf(colMeta.field)); + switch (column.getType()) { + case DATE: + return "'" + column.asString() + "'"; + case BYTES: + case STRING: + if (colMeta.type.equals("TIMESTAMP")) + return "\"" + column.asString() + "\""; + String value = column.asString(); + return "\'" + Utils.escapeSingleQuota(value) + "\'"; + case NULL: + case BAD: + return "NULL"; + case BOOL: + case DOUBLE: + case INT: + case LONG: + default: + return column.asString(); } } + + /** + * table: ["stb1"], column: ["ts", "f1", "f2", "t1"] + * data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, t1] => schemaless: stb1,t1=t1 f1=f1,f2=f2 ts + */ + private int writeBatchToSupTableBySchemaless(Connection conn, String table, List recordBatch) { + int count = 0; + TimestampPrecision timestampPrecision = schemaManager.loadDatabasePrecision(); + + List columnMetaList = this.columnMetas.get(table); + ColumnMeta ts = columnMetaList.stream().filter(colMeta -> colMeta.isPrimaryKey).findFirst().get(); + + List lines = new ArrayList<>(); + for (Record record : recordBatch) { + StringBuilder sb = new StringBuilder(); + sb.append(table).append(",") + .append(columnMetaList.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return colMeta.isTag; + }).map(colMeta -> { + String value = record.getColumn(indexOf(colMeta.field)).asString(); + if (value.contains(" ")) + value = value.replace(" ", "\\ "); + return colMeta.field + "=" + value; + }).collect(Collectors.joining(","))) + .append(" ") + .append(columnMetaList.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return !colMeta.isTag && !colMeta.isPrimaryKey; + }).map(colMeta -> { + return colMeta.field + "=" + buildSchemalessColumnValue(colMeta, record); +// return colMeta.field + "=" + record.getColumn(indexOf(colMeta.field)).asString(); + }).collect(Collectors.joining(","))) + .append(" "); + // timestamp + Column column = record.getColumn(indexOf(ts.field)); + Object tsValue = column.getRawData(); + if (column.getType() == Column.Type.DATE && tsValue instanceof Date) { + long time = column.asDate().getTime(); + switch (timestampPrecision) { + case NANOSEC: + sb.append(time * 1000000); + break; + case MICROSEC: + sb.append(time * 1000); + break; + case MILLISEC: + default: + sb.append(time); + } + } else if (column.getType() == Column.Type.STRING) { + sb.append(Utils.parseTimestamp(column.asString())); + } else { + sb.append(column.asLong()); + } + String line = sb.toString(); + LOG.debug(">>> " + line); + lines.add(line); + count++; + } + + SchemalessWriter writer = new SchemalessWriter(conn); + SchemalessTimestampType timestampType; + switch (timestampPrecision) { + case NANOSEC: + timestampType = SchemalessTimestampType.NANO_SECONDS; + break; + case MICROSEC: + timestampType = SchemalessTimestampType.MICRO_SECONDS; + break; + case MILLISEC: + timestampType = SchemalessTimestampType.MILLI_SECONDS; + break; + default: + timestampType = SchemalessTimestampType.NOT_CONFIGURED; + } + try { + writer.write(lines, SchemalessProtocolType.LINE, timestampType); + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + } + + LOG.warn("schemalessWriter does not return affected rows!"); + return count; + } + + private long dateAsLong(Column column) { + TimestampPrecision timestampPrecision = schemaManager.loadDatabasePrecision(); + long time = column.asDate().getTime(); + switch (timestampPrecision) { + case NANOSEC: + return time * 1000000; + case MICROSEC: + return time * 1000; + case MILLISEC: + default: + return time; + } + } + + private String buildSchemalessColumnValue(ColumnMeta colMeta, Record record) { + Column column = record.getColumn(indexOf(colMeta.field)); + switch (column.getType()) { + case DATE: + if (colMeta.type.equals("TIMESTAMP")) + return dateAsLong(column) + "i64"; + return "L'" + column.asString() + "'"; + case BYTES: + case STRING: + if (colMeta.type.equals("TIMESTAMP")) + return column.asString() + "i64"; + String value = column.asString(); + if (colMeta.type.startsWith("BINARY")) + return "'" + Utils.escapeSingleQuota(value) + "'"; + if (colMeta.type.startsWith("NCHAR")) + return "L'" + Utils.escapeSingleQuota(value) + "'"; + case NULL: + case BAD: + return "NULL"; + case DOUBLE: { + if (colMeta.type.equals("FLOAT")) + return column.asString() + "f32"; + if (colMeta.type.equals("DOUBLE")) + return column.asString() + "f64"; + } + case INT: + case LONG: { + if (colMeta.type.equals("TINYINT")) + return column.asString() + "i8"; + if (colMeta.type.equals("SMALLINT")) + return column.asString() + "i16"; + if (colMeta.type.equals("INT")) + return column.asString() + "i32"; + if (colMeta.type.equals("BIGINT")) + return column.asString() + "i64"; + } + case BOOL: + default: + return column.asString(); + } + } + + /** + * table: ["tb1"], column: [tbname, ts, f1, f2, t1] + * if contains("tbname") and tbname != tb1 continue; + * else if t1 != record[idx(t1)] or t2 != record[idx(t2)]... continue; + * else + * insert into tb1 (ts, f1, f2) values( record[idx(ts)], record[idx(f1)], record[idx(f2)]) + */ + private int writeBatchToSubTable(Connection conn, String table, List recordBatch) { + List columnMetas = this.columnMetas.get(table); + + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table).append(" ") + .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return !colMeta.isTag; + }).map(colMeta -> { + return colMeta.field; + }).collect(Collectors.joining(",", "(", ")"))) + .append(" values"); + int validRecords = 0; + for (Record record : recordBatch) { + if (columns.contains("tbname") && !table.equals(record.getColumn(indexOf("tbname")).asString())) + continue; + + boolean tagsAllMatch = columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return colMeta.isTag; + }).allMatch(colMeta -> { + return record.getColumn(indexOf(colMeta.field)).asString().equals(colMeta.value.toString()); + }); + + if (ignoreTagsUnmatched && !tagsAllMatch) + continue; + + sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + return !colMeta.isTag; + }).map(colMeta -> { + return buildColumnValue(colMeta, record); + }).collect(Collectors.joining(", ", "(", ") "))); + validRecords++; + } + + if (validRecords == 0) { + LOG.warn("no valid records in this batch"); + return 0; + } + + String sql = sb.toString(); + return executeUpdate(conn, sql); + } + + /** + * table: ["weather"], column: ["ts, f1, f2, f3, t1, t2"] + * sql: insert into weather (ts, f1, f2, f3, t1, t2) values( record[idx(ts), record[idx(f1)], ...) + */ + private int writeBatchToNormalTable(Connection conn, String table, List recordBatch) { + List columnMetas = this.columnMetas.get(table); + + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(table) + .append(" ") + .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).map(colMeta -> { + return colMeta.field; + }).collect(Collectors.joining(",", "(", ")"))) + .append(" values "); + + for (Record record : recordBatch) { + sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).map(colMeta -> { + return buildColumnValue(colMeta, record); + }).collect(Collectors.joining(",", "(", ")"))); + } + + String sql = sb.toString(); + return executeUpdate(conn, sql); + } + + private int indexOf(String colName) throws DataXException { + for (int i = 0; i < columns.size(); i++) { + if (columns.get(i).equals(colName)) + return i; + } + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, + "cannot find col: " + colName + " in columns: " + columns); + } + } \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java deleted file mode 100644 index 53ab9bb..0000000 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JDBCBatchWriter.java +++ /dev/null @@ -1,244 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import com.alibaba.datax.common.element.Column; -import com.alibaba.datax.common.element.Record; -import com.alibaba.datax.common.exception.DataXException; -import com.alibaba.datax.common.plugin.TaskPluginCollector; -import com.taosdata.jdbc.TSDBPreparedStatement; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * 使用JDBC原生写入接口批量写入。
- * 有两个限制条件导致批量写入的代码逻辑过于复杂，以至于需要开发新的类来封装。
- * 1. 用户必须提前把需要批量写入的数据搜集到ArrayList中 - * 2. 每批写入的表名必须相同。 - * 这个类的实现逻辑是： - * 1. 先把属于同一子表的Record缓存起来 - * 2. 缓存的数量达到batchSize阈值，自动执行一次批量写入 - * 3. 最后一批数据需要用户手动flush才能写入 - */ -public class JDBCBatchWriter { - public static final Logger LOG = LoggerFactory.getLogger(JDBCBatchWriter.class); - private TSDBPreparedStatement stmt; - private SchemaManager scm; - private Connection conn; - private int batchSize; - private TaskPluginCollector collector; - - // 缓存Record， key为tableName - Map> buf = new HashMap<>(); - // 缓存表的标签值， key为tableName - Map tableTagValues = new HashMap<>(); - private long sucCount = 0; - private final int tsColIndex; - private List fieldList; - // 每个record至少应该包含的列数，用于校验数据 - private int minColNum = 0; - private Map fieldIndexMap; - private List fieldTypes = null; - - public JDBCBatchWriter(Connection conn, TSDBPreparedStatement stmt, SchemaManager scm, int batchSize, TaskPluginCollector collector) { - this.conn = conn; - this.stmt = stmt; - this.scm = scm; - this.batchSize = batchSize; - this.collector = collector; - this.tsColIndex = scm.getTsColIndex(); - this.fieldList = scm.getFieldList(); - this.fieldIndexMap = scm.getFieldIndexMap(); - this.minColNum = 1 + fieldList.size() + scm.getDynamicTagCount(); - - } - - public void initFiledTypesAndTargetTable(List records) throws SQLException { - if (fieldTypes != null) { - return; - } - guessFieldTypes(records); - if (scm.shouldCreateTable()) { - scm.createSTable(conn, fieldTypes); - } - } - - public void append(Record record) throws SQLException { - int columnNum = record.getColumnNumber(); - if (columnNum < minColNum) { - // 实际列数小于期望列数 - collector.collectDirtyRecord(record, Msg.get("column_number_error")); - return; - } - String[] tagValues = scm.getTagValuesFromRecord(record); - if (tagValues == null) { - // 标签列包含null - collector.collectDirtyRecord(record, Msg.get("tag_value_error")); - return; - } - if (!scm.hasTimestamp(record)) { - // 时间戳列为null或类型错误 - collector.collectDirtyRecord(record, Msg.get("ts_value_error")); - return; - } - String tableName = scm.computeTableName(tagValues); - if (buf.containsKey(tableName)) { - List lis = buf.get(tableName); - lis.add(record); - if (lis.size() == batchSize) { - if (fieldTypes == null) { - initFiledTypesAndTargetTable(lis); - } - executeBatch(tableName); - lis.clear(); - } - } else { - List lis = new ArrayList<>(batchSize); - lis.add(record); - buf.put(tableName, lis); - tableTagValues.put(tableName, tagValues); - } - } - - /** - * 只有String类型比较特别，测试发现值为null的列会转成String类型。所以Column的类型为String并不代表这一列的类型真的是String。 - * - * @param records - */ - private void guessFieldTypes(List records) { - fieldTypes = new ArrayList<>(fieldList.size()); - for (int i = 0; i < fieldList.size(); ++i) { - int colIndex = fieldIndexMap.get(fieldList.get(i)); - boolean ok = false; - for (int j = 0; j < records.size() && !ok; ++j) { - Column column = records.get(j).getColumn(colIndex); - Column.Type type = column.getType(); - switch (type) { - case LONG: - case DOUBLE: - case DATE: - case BOOL: - case BYTES: - if (column.getRawData() != null) { - fieldTypes.add(type); - ok = true; - } - break; - case STRING: - // 只有非null且非空的String列，才会被真的当作String类型。 - String value = column.asString(); - if (value != null && !"".equals(value)) { - fieldTypes.add(type); - ok = true; - } - break; - default: - throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, fieldTypes.get(i).toString()); - } - } - if (!ok) { - // 根据采样的%d条数据，无法推断第%d列的数据类型 - throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, String.format(Msg.get("infer_column_type_error"), records.size(), i + 1)); - } - } - LOG.info("Field Types: {}", fieldTypes); - } - - /** - * 执行单表批量写入 - * - * @param tableName - * @throws SQLException - */ - private void executeBatch(String tableName) throws SQLException { - // 表名 - stmt.setTableName(tableName); - List records = buf.get(tableName); - // 标签 - String[] tagValues = tableTagValues.get(tableName); - LOG.debug("executeBatch {}", String.join(",", tagValues)); - for (int i = 0; i < tagValues.length; ++i) { - stmt.setTagNString(i, tagValues[i]); - } - // 时间戳 - ArrayList tsList = records.stream().map(r -> r.getColumn(tsColIndex).asDate().getTime()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setTimestamp(0, tsList); - // 字段 - for (int i = 0; i < fieldList.size(); ) { - String fieldName = fieldList.get(i); - int index = fieldIndexMap.get(fieldName); - switch (fieldTypes.get(i)) { - case LONG: - ArrayList lisLong = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setLong(++i, lisLong); - break; - case DOUBLE: - ArrayList lisDouble = records.stream().map(r -> r.getColumn(index).asDouble()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setDouble(++i, lisDouble); - break; - case STRING: - ArrayList lisString = records.stream().map(r -> r.getColumn(index).asString()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setNString(++i, lisString, 64); - break; - case DATE: - ArrayList lisTs = records.stream().map(r -> r.getColumn(index).asBigInteger().longValue()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setTimestamp(++i, lisTs); - break; - case BOOL: - ArrayList lisBool = records.stream().map(r -> r.getColumn(index).asBoolean()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setBoolean(++i, lisBool); - break; - case BYTES: - ArrayList lisBytes = records.stream().map(r -> r.getColumn(index).asString()).collect(Collectors.toCollection(ArrayList::new)); - stmt.setString(++i, lisBytes, 64); - break; - default: - throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, fieldTypes.get(i).toString()); - } - } - // 执行 - stmt.columnDataAddBatch(); - stmt.columnDataExecuteBatch(); - // 更新计数器 - sucCount += records.size(); - } - - /** - * 把缓存的Record全部写入 - */ - public void flush() throws SQLException { - if (fieldTypes == null) { - List records = new ArrayList<>(); - for (List lis : buf.values()) { - records.addAll(lis); - if (records.size() > 100) { - break; - } - } - if (records.size() > 0) { - initFiledTypesAndTargetTable(records); - } else { - return; - } - } - for (String tabName : buf.keySet()) { - if (buf.get(tabName).size() > 0) { - executeBatch(tabName); - } - } - stmt.columnDataCloseBatch(); - } - - /** - * @return 成功写入的数据量 - */ - public long getCount() { - return sucCount; - } -} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java deleted file mode 100644 index 0aabe32..0000000 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnection.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import java.util.Properties; - -public class JniConnection { - - private static final long JNI_NULL_POINTER = 0L; - private static final int JNI_SUCCESSFUL = 0; - public static final String PROPERTY_KEY_CONFIG_DIR = "cfgdir"; - public static final String PROPERTY_KEY_LOCALE = "locale"; - public static final String PROPERTY_KEY_CHARSET = "charset"; - public static final String PROPERTY_KEY_TIME_ZONE = "timezone"; - - private long conn; - - static { - System.loadLibrary("taos"); - } - - public JniConnection(Properties props) throws Exception { - initImp(props.getProperty(PROPERTY_KEY_CONFIG_DIR, null)); - - String locale = props.getProperty(PROPERTY_KEY_LOCALE); - if (setOptions(0, locale) < 0) { - throw new Exception("Failed to set locale: " + locale + ". System default will be used."); - } - String charset = props.getProperty(PROPERTY_KEY_CHARSET); - if (setOptions(1, charset) < 0) { - throw new Exception("Failed to set charset: " + charset + ". System default will be used."); - } - String timezone = props.getProperty(PROPERTY_KEY_TIME_ZONE); - if (setOptions(2, timezone) < 0) { - throw new Exception("Failed to set timezone: " + timezone + ". System default will be used."); - } - } - - public void open(String host, int port, String dbname, String user, String password) throws Exception { - if (this.conn != JNI_NULL_POINTER) { - close(); - this.conn = JNI_NULL_POINTER; - } - - this.conn = connectImp(host, port, dbname, user, password); - if (this.conn == JNI_NULL_POINTER) { - String errMsg = getErrMsgImp(0); - throw new Exception(errMsg); - } - } - - public void insertOpentsdbJson(String json) throws Exception { - if (this.conn == JNI_NULL_POINTER) { - throw new Exception("JNI connection is NULL"); - } - - long result = insertOpentsdbJson(json, this.conn); - int errCode = getErrCodeImp(this.conn, result); - if (errCode != JNI_SUCCESSFUL) { - String errMsg = getErrMsgImp(result); - freeResultSetImp(this.conn, result); - throw new Exception(errMsg); - } - freeResultSetImp(this.conn, result); - } - - public void close() throws Exception { - int code = this.closeConnectionImp(this.conn); - if (code != 0) { - throw new Exception("JNI closeConnection failed"); - } - this.conn = JNI_NULL_POINTER; - } - - private static native void initImp(String configDir); - - private static native int setOptions(int optionIndex, String optionValue); - - private native long connectImp(String host, int port, String dbName, String user, String password); - - private native int getErrCodeImp(long connection, long pSql); - - private native String getErrMsgImp(long pSql); - - private native void freeResultSetImp(long connection, long pSql); - - private native int closeConnectionImp(long connection); - - private native long insertOpentsdbJson(String json, long pSql); - -} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index 7fb383e..1d7ee21 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -1,14 +1,12 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; public class Key { - public static final String HOST = "host"; - public static final String PORT = "port"; - public static final String DBNAME = "dbName"; - public static final String USER = "username"; + public static final String USERNAME = "username"; public static final String PASSWORD = "password"; + public static final String CONNECTION = "connection"; public static final String BATCH_SIZE = "batchSize"; - public static final String STABLE = "stable"; - public static final String TAG_COLUMN = "tagColumn"; - public static final String FIELD_COLUMN = "fieldColumn"; - public static final String TIMESTAMP_COLUMN = "timestampColumn"; -} + public static final String TABLE = "table"; + public static final String JDBC_URL = "jdbcUrl"; + public static final String COLUMN = "column"; + public static final String IGNORE_TAGS_UNMATCHED = "ignoreTagsUnmatched"; +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java index e1b8f5d..2fb5a98 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/OpentsdbDataHandler.java @@ -5,50 +5,50 @@ import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.alibaba.datax.common.util.Configuration; +import com.taosdata.jdbc.SchemalessWriter; +import com.taosdata.jdbc.enums.SchemalessProtocolType; +import com.taosdata.jdbc.enums.SchemalessTimestampType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Properties; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; public class OpentsdbDataHandler implements DataHandler { private static final Logger LOG = LoggerFactory.getLogger(OpentsdbDataHandler.class); - private static final String DEFAULT_BATCH_SIZE = "1"; + private SchemalessWriter writer; + + private String jdbcUrl; + private String user; + private String password; + int batchSize; + + public OpentsdbDataHandler(Configuration config) { + // opentsdb json protocol use JNI and schemaless API to write + this.jdbcUrl = config.getString(Key.JDBC_URL); + this.user = config.getString(Key.USERNAME, "root"); + this.password = config.getString(Key.PASSWORD, "taosdata"); + this.batchSize = config.getInt(Key.BATCH_SIZE, Constants.DEFAULT_BATCH_SIZE); + } @Override - public long handle(RecordReceiver lineReceiver, Properties properties, TaskPluginCollector collector) { - // opentsdb json protocol use JNI and schemaless API to write - String host = properties.getProperty(Key.HOST); - int port = Integer.parseInt(properties.getProperty(Key.PORT)); - String dbname = properties.getProperty(Key.DBNAME); - String user = properties.getProperty(Key.USER); - String password = properties.getProperty(Key.PASSWORD); - - JniConnection conn = null; - long count = 0; - try { - conn = new JniConnection(properties); - conn.open(host, port, dbname, user, password); - LOG.info("TDengine connection established, host: " + host + ", port: " + port + ", dbname: " + dbname + ", user: " + user); - int batchSize = Integer.parseInt(properties.getProperty(Key.BATCH_SIZE, DEFAULT_BATCH_SIZE)); - count = writeOpentsdb(lineReceiver, conn, batchSize); + public int handle(RecordReceiver lineReceiver, TaskPluginCollector collector) { + int count = 0; + try (Connection conn = DriverManager.getConnection(jdbcUrl, user, password);) { + LOG.info("connection[ jdbcUrl: " + jdbcUrl + ", username: " + user + "] established."); + writer = new SchemalessWriter(conn); + count = write(lineReceiver, batchSize); } catch (Exception e) { - LOG.error(e.getMessage()); - e.printStackTrace(); - } finally { - try { - if (conn != null) - conn.close(); - } catch (Exception e) { - e.printStackTrace(); - } - LOG.info("TDengine connection closed"); + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); } return count; } - private long writeOpentsdb(RecordReceiver lineReceiver, JniConnection conn, int batchSize) { - long recordIndex = 1; + private int write(RecordReceiver lineReceiver, int batchSize) throws DataXException { + int recordIndex = 1; try { Record record; StringBuilder sb = new StringBuilder(); @@ -56,14 +56,14 @@ public class OpentsdbDataHandler implements DataHandler { if (batchSize == 1) { String jsonData = recordToString(record); LOG.debug(">>> " + jsonData); - conn.insertOpentsdbJson(jsonData); + writer.write(jsonData, SchemalessProtocolType.JSON, SchemalessTimestampType.NOT_CONFIGURED); } else if (recordIndex % batchSize == 1) { sb.append("[").append(recordToString(record)).append(","); } else if (recordIndex % batchSize == 0) { sb.append(recordToString(record)).append("]"); String jsonData = sb.toString(); LOG.debug(">>> " + jsonData); - conn.insertOpentsdbJson(jsonData); + writer.write(jsonData, SchemalessProtocolType.JSON, SchemalessTimestampType.NOT_CONFIGURED); sb.delete(0, sb.length()); } else { sb.append(recordToString(record)).append(","); @@ -72,11 +72,11 @@ public class OpentsdbDataHandler implements DataHandler { } if (sb.length() != 0 && sb.charAt(0) == '[') { String jsonData = sb.deleteCharAt(sb.length() - 1).append("]").toString(); + System.err.println(jsonData); LOG.debug(">>> " + jsonData); - conn.insertOpentsdbJson(jsonData); + writer.write(jsonData, SchemalessProtocolType.JSON, SchemalessTimestampType.NOT_CONFIGURED); } } catch (Exception e) { - LOG.error("TDengineWriter ERROR: " + e.getMessage()); throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e); } return recordIndex - 1; diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index d67a658..877b9b6 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -1,9 +1,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; -import com.alibaba.datax.common.element.Column; -import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.exception.DataXException; -import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,260 +10,163 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.*; -import java.util.stream.Collectors; public class SchemaManager { private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class); - private String stable; // 目标超表名 - private Map fixedTagValue = new HashMap<>(); // 固定标签值 标签名 -> 标签值 - private Map tagIndexMap = new HashMap<>(); // 动态标签值 标签名 -> 列索引 - private Map fieldIndexMap = new HashMap<>(); // 字段名 -> 字段索引 - private String tsColName; // 时间戳列名 - private int tsColIndex = -1; // 时间戳列索引 - private List fieldList = new ArrayList<>(); - private List tagList = new ArrayList<>(); - private boolean canInferSchemaFromConfig = false; + private final Connection conn; + private TimestampPrecision precision; - - public SchemaManager() { + SchemaManager(Connection conn) { + this.conn = conn; } - public SchemaManager(Properties properties) { - getFromConfig(properties); - } + public TimestampPrecision loadDatabasePrecision() throws DataXException { + if (this.precision != null) + return this.precision; - private String mapDataxType(Column.Type type) { - switch (type) { - case LONG: - return "BIGINT"; - case DOUBLE: - return "DOUBLE"; - case STRING: - return "NCHAR(64)"; - case DATE: - return "TIMESTAMP"; - case BOOL: - return "BOOL"; - case BYTES: - return "BINARY(64)"; - default: - throw DataXException.asDataXException(TDengineWriterErrorCode.TYPE_ERROR, type.toString()); - } - } - - public void setStable(String stable) { - stable = stable; - } - - public String getStable() { - return stable; - } - - private void getFromConfig(Properties properties) { - stable = properties.getProperty(Key.STABLE); - if (stable == null) { - LOG.error("Config error: no stable"); - return; - } - for (Object key : properties.keySet()) { - String k = (String) key; - String v = properties.getProperty(k); - - String[] ps = k.split("\\."); - if (ps.length == 1) { - continue; - } - if (k.startsWith(Key.TAG_COLUMN)) { - String tagName = ps[1]; - try { - Integer tagIndex = Integer.parseInt(v); - this.tagIndexMap.put(tagName, tagIndex); - tagList.add(tagName); - } catch (NumberFormatException e) { - fixedTagValue.put(tagName, v); - tagList.add(tagName); - } - } else if (k.startsWith(Key.FIELD_COLUMN)) { - String fieldName = ps[1]; - Integer fileIndex = Integer.parseInt(v); - fieldIndexMap.put(fieldName, fileIndex); - } else if (k.startsWith(Key.TIMESTAMP_COLUMN)) { - tsColName = ps[1]; - tsColIndex = Integer.parseInt(v); - } - } - List sortedFieldName = fieldIndexMap.entrySet().stream().sorted((x, y) -> x.getValue().compareTo(y.getValue())).map(e -> e.getKey()).collect(Collectors.toList()); - fieldList.addAll(sortedFieldName); // 排序的目的是保证自动建表时列的顺序和输入数据的列的顺序保持一致 - canInferSchemaFromConfig = tsColIndex > -1 && !(fixedTagValue.isEmpty() && tagIndexMap.isEmpty()) && !fieldIndexMap.isEmpty(); - LOG.info("Config file parsed result：fixedTags=[{}] ,tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", fixedTagValue.keySet()), String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); - } - - public boolean shouldGuessSchema() { - return !canInferSchemaFromConfig; - } - - public boolean shouldCreateTable() { - return canInferSchemaFromConfig; - } - - public boolean configValid() { - boolean valid = (tagList.size() > 0 && fieldList.size() > 0 && tsColIndex > -1) || (tagList.size() == 0 && fieldList.size() == 0 && tsColIndex == -1); - if (!valid) { - LOG.error("Config error: tagColumn, fieldColumn and timestampColumn must be present together or absent together."); - } - return valid; - } - - /** - * 通过执行`describe dbname.stable`命令，获取表的schema.
- * describe命名返回有4列内容，分布是：Field,Type,Length,Note
- * - * @return 成功返回true，如果超表不存在或其他错误则返回false - */ - public boolean getFromDB(Connection conn) { - try { - List stables = getSTables(conn); - if (!stables.contains(stable)) { - LOG.error("super table {} not exist， fail to get schema from database.", stable); - return false; - } - } catch (SQLException e) { - LOG.error(e.getMessage()); - e.printStackTrace(); - return false; - } try (Statement stmt = conn.createStatement()) { - ResultSet rs = stmt.executeQuery("describe " + stable); - int colIndex = 0; + ResultSet rs = stmt.executeQuery("select database()"); + String dbname = null; while (rs.next()) { - String name = rs.getString(1); - String type = rs.getString(2); - String note = rs.getString(4); - if ("TIMESTAMP".equals(type)) { - tsColName = name; - tsColIndex = colIndex; - } else if ("TAG".equals(note)) { - tagIndexMap.put(name, colIndex); - tagList.add(name); - } else { - fieldIndexMap.put(name, colIndex); - fieldList.add(name); + dbname = rs.getString("database()"); + } + if (dbname == null) + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, + "Database not specified or available"); + + rs = stmt.executeQuery("show databases"); + while (rs.next()) { + String name = rs.getString("name"); + if (!name.equalsIgnoreCase(dbname)) + continue; + String precision = rs.getString("precision"); + switch (precision) { + case "ns": + this.precision = TimestampPrecision.NANOSEC; + break; + case "us": + this.precision = TimestampPrecision.MICROSEC; + break; + case "ms": + default: + this.precision = TimestampPrecision.MILLISEC; } - colIndex++; } - LOG.info("table info：tags=[{}], fields=[{}], tsColName={}, tsIndex={}", String.join(",", tagIndexMap.keySet()), String.join(",", fieldList), tsColName, tsColIndex); - return true; } catch (SQLException e) { - LOG.error(e.getMessage()); - e.printStackTrace(); - return false; + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); } + return this.precision; } - public static List getSTables(Connection conn) throws SQLException { - List stables = new ArrayList<>(); + public Map loadTableMeta(List tables) throws DataXException { + Map tableMetas = new HashMap(); + try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery("show stables"); while (rs.next()) { - String name = rs.getString(1); - stables.add(name); + TableMeta tableMeta = buildSupTableMeta(rs); + if (!tables.contains(tableMeta.tbname)) + continue; + tableMetas.put(tableMeta.tbname, tableMeta); } - } - return stables; - } - public void createSTable(Connection conn, List fieldTypes) throws SQLException { - StringBuilder sb = new StringBuilder(); - sb.append("CREATE STABLE IF NOT EXISTS ").append(stable).append("("); - sb.append(tsColName).append(" ").append("TIMESTAMP,"); - for (int i = 0; i < fieldList.size(); ++i) { - String fieldName = fieldList.get(i); - Column.Type dxType = fieldTypes.get(i); - sb.append(fieldName).append(' '); - String tdType = mapDataxType(dxType); - sb.append(tdType).append(','); - } - sb.deleteCharAt(sb.length() - 1); - sb.append(") TAGS("); - for (String tagName : tagList) { - sb.append(tagName).append(" NCHAR(64),"); - } - sb.deleteCharAt(sb.length() - 1); - sb.append(")"); - String q = sb.toString(); - LOG.info("run sql：" + q); - try (Statement stmt = conn.createStatement()) { - stmt.execute(q); - } - } - - public String[] getTagValuesFromRecord(Record record) { - String[] tagValues = new String[tagList.size()]; - for (int i = 0; i < tagList.size(); ++i) { - if (fixedTagValue.containsKey(tagList.get(i))) { - tagValues[i] = fixedTagValue.get(tagList.get(i)); - } else { - int tagIndex = tagIndexMap.get(tagList.get(i)); - tagValues[i] = record.getColumn(tagIndex).asString(); + rs = stmt.executeQuery("show tables"); + while (rs.next()) { + TableMeta tableMeta = buildSubTableMeta(rs); + if (!tables.contains(tableMeta.tbname)) + continue; + tableMetas.put(tableMeta.tbname, tableMeta); } - if (tagValues[i] == null) { - return null; + + for (String tbname : tables) { + if (!tableMetas.containsKey(tbname)) { + LOG.error("table metadata of " + tbname + " is empty!"); + } } + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); } - return tagValues; + return tableMetas; } - public boolean hasTimestamp(Record record) { - Column column = record.getColumn(tsColIndex); - if (column.getType() == Column.Type.DATE && column.asDate() != null) { - return true; - } else { - return false; + public Map> loadColumnMetas(List tables) throws DataXException { + Map> ret = new HashMap<>(); + + for (String table : tables) { + List columnMetaList = new ArrayList<>(); + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("describe " + table); + for (int i = 0; rs.next(); i++) { + ColumnMeta columnMeta = buildColumnMeta(rs, i == 0); + columnMetaList.add(columnMeta); + } + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); + } + + if (columnMetaList.isEmpty()) { + LOG.error("column metadata of " + table + " is empty!"); + continue; + } + + columnMetaList.stream().filter(colMeta -> colMeta.isTag).forEach(colMeta -> { + String sql = "select " + colMeta.field + " from " + table; + Object value = null; + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery(sql); + for (int i = 0; rs.next(); i++) { + value = rs.getObject(colMeta.field); + if (i > 0) { + value = null; + break; + } + } + } catch (SQLException e) { + e.printStackTrace(); + } + colMeta.value = value; + }); + + LOG.debug("load column metadata of " + table + ": " + Arrays.toString(columnMetaList.toArray())); + ret.put(table, columnMetaList); } + return ret; } - public Map getFieldIndexMap() { - return fieldIndexMap; + private TableMeta buildSupTableMeta(ResultSet rs) throws SQLException { + TableMeta tableMeta = new TableMeta(); + tableMeta.tableType = TableType.SUP_TABLE; + tableMeta.tbname = rs.getString("name"); + tableMeta.columns = rs.getInt("columns"); + tableMeta.tags = rs.getInt("tags"); + tableMeta.tables = rs.getInt("tables"); + + LOG.debug("load table metadata of " + tableMeta.tbname + ": " + tableMeta); + return tableMeta; } - public List getFieldList() { - return fieldList; + private TableMeta buildSubTableMeta(ResultSet rs) throws SQLException { + TableMeta tableMeta = new TableMeta(); + String stable_name = rs.getString("stable_name"); + tableMeta.tableType = StringUtils.isBlank(stable_name) ? TableType.NML_TABLE : TableType.SUB_TABLE; + tableMeta.tbname = rs.getString("table_name"); + tableMeta.columns = rs.getInt("columns"); + tableMeta.stable_name = StringUtils.isBlank(stable_name) ? null : stable_name; + + LOG.debug("load table metadata of " + tableMeta.tbname + ": " + tableMeta); + return tableMeta; } - public String getJoinedFieldNames() { - return tsColName + ", " + String.join(", ", fieldList); + private ColumnMeta buildColumnMeta(ResultSet rs, boolean isPrimaryKey) throws SQLException { + ColumnMeta columnMeta = new ColumnMeta(); + columnMeta.field = rs.getString("Field"); + columnMeta.type = rs.getString("Type"); + columnMeta.length = rs.getInt("Length"); + columnMeta.note = rs.getString("Note"); + columnMeta.isTag = columnMeta.note != null && columnMeta.note.equals("TAG"); + columnMeta.isPrimaryKey = isPrimaryKey; + return columnMeta; } - public int getTsColIndex() { - return tsColIndex; - } - - public String getTagValuesPlaceHolder() { - return tagList.stream().map(x -> "?").collect(Collectors.joining(",")); - } - - public String getFieldValuesPlaceHolder() { - return "?, " + fieldList.stream().map(x -> "?").collect(Collectors.joining(", ")); - } - - /** - * 计算子表表名 - *

    - *
  1. 将标签的value 组合成为如下的字符串: tag_value1!tag_value2!tag_value3。
    2. - *
  2. 计算该字符串的 MD5 散列值 "md5_val"。
    3. - *
  3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。
    4. - *
- * - * @param tagValues - * @return - */ - public String computeTableName(String[] tagValues) { - String s = String.join("!", tagValues); - return "t_" + DigestUtils.md5Hex(s); - } - - public int getDynamicTagCount() { - return tagIndexMap.size(); - } } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java index 79e5238..7cc76a7 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -1,37 +1,56 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; - +import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; -import java.util.Properties; -import java.util.Set; public class TDengineWriter extends Writer { private static final String PEER_PLUGIN_NAME = "peerPluginName"; - static { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - public static class Job extends Writer.Job { private Configuration originalConfig; + private static final Logger LOG = LoggerFactory.getLogger(Job.class); @Override public void init() { this.originalConfig = super.getPluginJobConf(); this.originalConfig.set(PEER_PLUGIN_NAME, getPeerPluginName()); + + // check user + String user = this.originalConfig.getString(Key.USERNAME); + if (StringUtils.isBlank(user)) + throw DataXException.asDataXException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + + Key.USERNAME + "] is not set."); + + // check password + String password = this.originalConfig.getString(Key.PASSWORD); + if (StringUtils.isBlank(password)) + throw DataXException.asDataXException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + + Key.PASSWORD + "] is not set."); + + // check connection + List connection = this.originalConfig.getList(Key.CONNECTION); + if (connection == null || connection.isEmpty()) + throw DataXException.asDataXException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + + Key.CONNECTION + "] is not set."); + if (connection.size() > 1) + LOG.warn("connection.size is " + connection.size() + " and only connection[0] will be used."); + Configuration conn = Configuration.from(connection.get(0).toString()); + String jdbcUrl = conn.getString(Key.JDBC_URL); + if (StringUtils.isBlank(jdbcUrl)) + throw DataXException.asDataXException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" + + Key.JDBC_URL + "] of connection is not set."); + + // check column } @Override @@ -42,22 +61,30 @@ public class TDengineWriter extends Writer { @Override public List split(int mandatoryNumber) { List writerSplitConfigs = new ArrayList<>(); + + List conns = this.originalConfig.getList(Key.CONNECTION); for (int i = 0; i < mandatoryNumber; i++) { - writerSplitConfigs.add(this.originalConfig); + Configuration clone = this.originalConfig.clone(); + Configuration conf = Configuration.from(conns.get(0).toString()); + String jdbcUrl = conf.getString(Key.JDBC_URL); + clone.set(Key.JDBC_URL, jdbcUrl); + clone.set(Key.TABLE, conf.getList(Key.TABLE)); + clone.remove(Key.CONNECTION); + writerSplitConfigs.add(clone); } + return writerSplitConfigs; } } public static class Task extends Writer.Task { - private static final Logger LOG = LoggerFactory.getLogger(Job.class); + private static final Logger LOG = LoggerFactory.getLogger(Task.class); private Configuration writerSliceConfig; @Override public void init() { this.writerSliceConfig = getPluginJobConf(); - } @Override @@ -67,23 +94,16 @@ public class TDengineWriter extends Writer { @Override public void startWrite(RecordReceiver lineReceiver) { - Set keys = this.writerSliceConfig.getKeys(); - Properties properties = new Properties(); - for (String key : keys) { - String value = this.writerSliceConfig.getString(key); - properties.setProperty(key, value); - } - if (!keys.contains(Key.USER)) { - properties.setProperty(Key.USER, "root"); - } - if (!keys.contains(Key.PASSWORD)) { - properties.setProperty(Key.PASSWORD, "taosdata"); - } - LOG.debug("========================properties==========================\n" + properties); String peerPluginName = this.writerSliceConfig.getString(PEER_PLUGIN_NAME); LOG.debug("start to handle record from: " + peerPluginName); - DataHandler handler = DataHandlerFactory.build(peerPluginName); - long records = handler.handle(lineReceiver, properties, getTaskPluginCollector()); + + DataHandler handler; + if (peerPluginName.equals("opentsdbreader")) + handler = new OpentsdbDataHandler(this.writerSliceConfig); + else + handler = new DefaultDataHandler(this.writerSliceConfig); + + long records = handler.handle(lineReceiver, getTaskPluginCollector()); LOG.debug("handle data finished, records: " + records); } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java index 994f1e8..469449e 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java @@ -3,13 +3,16 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.spi.ErrorCode; public enum TDengineWriterErrorCode implements ErrorCode { - RUNTIME_EXCEPTION("TDengineWriter-00", "运行时异常"), - TYPE_ERROR("TDengineWriter-00", "Datax类型无法正确映射到TDengine类型"); + + REQUIRED_VALUE("TDengineWriter-00", "缺失必要的值"), + ILLEGAL_VALUE("TDengineWriter-01", "值非法"), + RUNTIME_EXCEPTION("TDengineWriter-02", "运行时异常"), + TYPE_ERROR("TDengineWriter-03", "Datax类型无法正确映射到TDengine类型"); private final String code; private final String description; - private TDengineWriterErrorCode(String code, String description) { + TDengineWriterErrorCode(String code, String description) { this.code = code; this.description = description; } @@ -26,7 +29,6 @@ public enum TDengineWriterErrorCode implements ErrorCode { @Override public String toString() { - return String.format("Code:[%s], Description:[%s]. ", this.code, - this.description); + return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); } } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableMeta.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableMeta.java new file mode 100644 index 0000000..2a6b053 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableMeta.java @@ -0,0 +1,22 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public class TableMeta { + TableType tableType; + String tbname; + int columns; + int tags; + int tables; + String stable_name; + + @Override + public String toString() { + return "TableMeta{" + + "tableType=" + tableType + + ", tbname='" + tbname + '\'' + + ", columns=" + columns + + ", tags=" + tags + + ", tables=" + tables + + ", stable_name='" + stable_name + '\'' + + '}'; + } +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableType.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableType.java new file mode 100644 index 0000000..6c97ac3 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TableType.java @@ -0,0 +1,5 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public enum TableType { + SUP_TABLE, SUB_TABLE, NML_TABLE +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TimestampPrecision.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TimestampPrecision.java new file mode 100644 index 0000000..46564d6 --- /dev/null +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TimestampPrecision.java @@ -0,0 +1,5 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public enum TimestampPrecision { + MILLISEC, MICROSEC, NANOSEC +} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h deleted file mode 100644 index 4bdf363..0000000 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/com_alibaba_datax_plugin_writer_tdenginewriter_JniConnection.h +++ /dev/null @@ -1,105 +0,0 @@ -/* DO NOT EDIT THIS FILE - it is machine generated */ -#include -/* Header for class com_alibaba_datax_plugin_writer_JniConnection */ - -#ifndef _Included_com_alibaba_datax_plugin_writer_JniConnection -#define _Included_com_alibaba_datax_plugin_writer_JniConnection -#ifdef __cplusplus -extern "C" { -#endif -#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER -#define com_alibaba_datax_plugin_writer_JniConnection_JNI_NULL_POINTER 0LL -#undef com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL -#define com_alibaba_datax_plugin_writer_JniConnection_JNI_SUCCESSFUL 0L -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: initImp - * Signature: (Ljava/lang/String;)V - */ -JNIEXPORT void JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_initImp - (JNIEnv *, jclass, jstring); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: setOptions - * Signature: (ILjava/lang/String;)I - */ -JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_setOptions - (JNIEnv *, jclass, jint, jstring); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: getTsCharset - * Signature: ()Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getTsCharset - (JNIEnv *, jclass); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: connectImp - * Signature: (Ljava/lang/String;ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;)J - */ -JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_connectImp - (JNIEnv *, jobject, jstring, jint, jstring, jstring, jstring); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: executeQueryImp - * Signature: ([BJ)J - */ -JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_executeQueryImp - (JNIEnv *, jobject, jbyteArray, jlong); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: getErrCodeImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrCodeImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: getErrMsgImp - * Signature: (J)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: getErrMsgByCode - * Signature: (J)Ljava/lang/String; - */ -JNIEXPORT jstring JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getErrMsgByCode - (JNIEnv *, jobject, jlong); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: getAffectedRowsImp - * Signature: (JJ)I - */ -JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_getAffectedRowsImp - (JNIEnv *, jobject, jlong, jlong); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: closeConnectionImp - * Signature: (J)I - */ -JNIEXPORT jint JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_closeConnectionImp - (JNIEnv *, jobject, jlong); - -/* - * Class: com_alibaba_datax_plugin_writer_JniConnection - * Method: insertOpentsdbJson - * Signature: (Ljava/lang/String;J)J - */ -JNIEXPORT jlong JNICALL Java_com_alibaba_datax_plugin_writer_JniConnection_insertOpentsdbJson - (JNIEnv *, jobject, jstring, jlong); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java new file mode 100644 index 0000000..733698a --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java @@ -0,0 +1,235 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.DateColumn; +import com.alibaba.datax.common.element.LongColumn; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.element.StringColumn; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.core.transport.record.DefaultRecord; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class DefaultDataHandlerTest { + + private static final String host = "192.168.56.105"; + private static Connection conn; + + @Test + public void writeSupTableBySQL() { + // given + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"tbname\", \"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"table\":[\"stb1\"]," + + "\"jdbcUrl\":\"jdbc:TAOS-RS://" + host + ":6041/test\"," + + "\"batchSize\": \"1000\"" + + "}"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new StringColumn("tb" + (i + 10))); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(2)); + record.addColumn(new LongColumn(i)); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(conn); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + + int count = handler.writeBatch(conn, recordList); + + // then + Assert.assertEquals(10, count); + } + + @Test + public void writeSupTableBySQL_2() { + // given + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"tbname\", \"ts\", \"f1\", \"t1\"]," + + "\"table\":[\"stb1\"]," + + "\"jdbcUrl\":\"jdbc:TAOS-RS://" + host + ":6041/test\"," + + "\"batchSize\": \"1000\"" + + "}"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new StringColumn("tb" + (i + 10))); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(i)); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(conn); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + + int count = handler.writeBatch(conn, recordList); + + // then + Assert.assertEquals(10, count); + } + + @Test + public void writeSubTableWithTableName() { + // given + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"tbname\", \"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"table\":[\"tb1\"]," + + "\"jdbcUrl\":\"jdbc:TAOS-RS://" + host + ":6041/test\"," + + "\"batchSize\": \"1000\"" + + "}"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new StringColumn("tb" + i)); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(2)); + record.addColumn(new LongColumn(i)); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(conn); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + + int count = handler.writeBatch(conn, recordList); + + // then + Assert.assertEquals(1, count); + } + + @Test + public void writeSubTableWithoutTableName() { + // given + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"table\":[\"tb1\"]," + + "\"jdbcUrl\":\"jdbc:TAOS-RS://" + host + ":6041/test\"," + + "\"batchSize\": \"1000\"," + + "\"ignoreTagsUnmatched\": \"true\"" + + "}"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(2)); + record.addColumn(new LongColumn(i)); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(conn); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + + int count = handler.writeBatch(conn, recordList); + + // then + Assert.assertEquals(1, count); + } + + @Test + public void writeNormalTable() { + // given + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"table\":[\"weather\"]," + + "\"jdbcUrl\":\"jdbc:TAOS-RS://" + host + ":6041/test\"," + + "\"batchSize\": \"1000\"," + + "\"ignoreTagsUnmatched\": \"true\"" + + "}"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(2)); + record.addColumn(new LongColumn(i)); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(conn); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + + int count = handler.writeBatch(conn, recordList); + + // then + Assert.assertEquals(10, count); + } + + @BeforeClass + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041", "root", "taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists scm_test"); + stmt.execute("create database if not exists scm_test"); + stmt.execute("use scm_test"); + stmt.execute("create table stb1(ts timestamp, f1 int, f2 int) tags(t1 nchar(32))"); + stmt.execute("create table stb2(ts timestamp, f1 int, f2 int, f3 int) tags(t1 int, t2 int)"); + stmt.execute("create table tb1 using stb1 tags(1)"); + stmt.execute("create table tb2 using stb1 tags(2)"); + stmt.execute("create table tb3 using stb2 tags(1,1)"); + stmt.execute("create table tb4 using stb2 tags(2,2)"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 int, f3 int, t1 int, t2 int)"); + } + } + + @AfterClass + public static void afterClass() throws SQLException { + if (conn != null) { + conn.close(); + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java new file mode 100644 index 0000000..48baad1 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java @@ -0,0 +1,85 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.common.element.DateColumn; +import com.alibaba.datax.common.element.LongColumn; +import com.alibaba.datax.common.element.Record; +import com.alibaba.datax.common.element.StringColumn; +import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.core.transport.record.DefaultRecord; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class DefaultDataHandlerTest2 { + + private static final String host = "192.168.1.93"; + private static Connection conn; + + @Test + public void writeSupTableBySchemaless() throws SQLException { + // given + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"table\":[\"stb1\"]," + + "\"jdbcUrl\":\"jdbc:TAOS://" + host + ":6030/scm_test\"," + + "\"batchSize\": \"1000\"" + + "}"); + + String jdbcUrl = configuration.getString("jdbcUrl"); + Connection connection = DriverManager.getConnection(jdbcUrl, "root", "taosdata"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(2)); + record.addColumn(new StringColumn("t" + i + " 22")); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(connection); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); + + int count = handler.writeBatch(connection, recordList); + + // then + Assert.assertEquals(10, count); + } + + @BeforeClass + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041", "root", "taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists scm_test"); + stmt.execute("create database if not exists scm_test"); + stmt.execute("use scm_test"); + stmt.execute("create table stb1(ts timestamp, f1 int, f2 int) tags(t1 nchar(32))"); + } + } + + @AfterClass + public static void afterClass() throws SQLException { + if (conn != null) { + conn.close(); + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java new file mode 100644 index 0000000..985c804 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java @@ -0,0 +1,65 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.*; + +public class EngineTest { + + @Test + public void opentsdb2tdengine() throws SQLException { + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/opentsdb2tdengine.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + try { + Engine.entry(params); + } catch (Throwable e) { + e.printStackTrace(); + } + + // assert + String jdbcUrl = "jdbc:TAOS://192.168.56.105:6030/test?timestampFormat=TIMESTAMP"; + try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("select count(*) from weather_temperature"); + int rows = 0; + while (rs.next()) { + rows = rs.getInt("count(*)"); + } + Assert.assertEquals(5, rows); + stmt.close(); + } + } + + @Test + public void mysql2tdengine() { + // given + // MYSQL SQL: + // create table t(id int primary key AUTO_INCREMENT, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double,ts timestamp, dt datetime,f7 nchar(64)) + // insert into t(f1,f2,f3,f4,f5,f6,ts,dt,f7) values(1,2,3,4,5,6,'2022-01-28 12:00:00','2022-01-28 12:00:00', 'beijing'); + // TDengine SQL: + // create table t(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double, dt timestamp,f7 nchar(64)); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/mysql2tdengine.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + try { + Engine.entry(params); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + @Test + public void tdengine2tdengine() { + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/tdengine2tdengine.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + try { + Engine.entry(params); + } catch (Throwable e) { + e.printStackTrace(); + } + } +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java deleted file mode 100644 index 09c3df2..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/JniConnectionTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import org.junit.Test; - -import java.util.Properties; - -public class JniConnectionTest { - - @Test - public void test() throws Exception { - JniConnection connection = new JniConnection(new Properties()); - - connection.open("192.168.56.105", 6030, "test", "root", "taosdata"); - - String json = "{\"metric\":\"weather_temperature\",\"timestamp\":1609430400000,\"value\":123,\"tags\":{\"location\":\"beijing\",\"id\":\"t123\"}}"; - connection.insertOpentsdbJson(json); - - connection.close(); - } - -} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java new file mode 100644 index 0000000..6df0de1 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java @@ -0,0 +1,88 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.fastjson.util.TypeUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class SchemaManagerTest { + + private static Connection conn; + + @Test + public void loadTableMeta() throws SQLException { + // given + SchemaManager schemaManager = new SchemaManager(conn); + List tables = Arrays.asList("stb1", "stb2", "tb1", "tb3", "weather"); + + // when + Map tableMetaMap = schemaManager.loadTableMeta(tables); + + // then + TableMeta stb1 = tableMetaMap.get("stb1"); + Assert.assertEquals(TableType.SUP_TABLE, stb1.tableType); + Assert.assertEquals("stb1", stb1.tbname); + Assert.assertEquals(3, stb1.columns); + Assert.assertEquals(1, stb1.tags); + Assert.assertEquals(2, stb1.tables); + + TableMeta tb3 = tableMetaMap.get("tb3"); + Assert.assertEquals(TableType.SUB_TABLE, tb3.tableType); + Assert.assertEquals("tb3", tb3.tbname); + Assert.assertEquals(4, tb3.columns); + Assert.assertEquals("stb2", tb3.stable_name); + + TableMeta weather = tableMetaMap.get("weather"); + Assert.assertEquals(TableType.NML_TABLE, weather.tableType); + Assert.assertEquals("weather", weather.tbname); + Assert.assertEquals(6, weather.columns); + Assert.assertNull(weather.stable_name); + } + + @Test + public void loadColumnMetas() { + // given + SchemaManager schemaManager = new SchemaManager(conn); + List tables = Arrays.asList("stb1", "stb2", "tb1", "tb3", "weather"); + + // when + Map> columnMetaMap = schemaManager.loadColumnMetas(tables); + + // then + List stb1 = columnMetaMap.get("stb1"); + Assert.assertEquals(4, stb1.size()); + } + + @BeforeClass + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://192.168.56.105:6041", "root", "taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists scm_test"); + stmt.execute("create database if not exists scm_test"); + stmt.execute("use scm_test"); + stmt.execute("create table stb1(ts timestamp, f1 int, f2 int) tags(t1 int)"); + stmt.execute("create table stb2(ts timestamp, f1 int, f2 int, f3 int) tags(t1 int, t2 int)"); + stmt.execute("insert into tb1 using stb1 tags(1) values(now, 1, 2)"); + stmt.execute("insert into tb2 using stb1 tags(2) values(now, 1, 2)"); + stmt.execute("insert into tb3 using stb2 tags(1,1) values(now, 1, 2, 3)"); + stmt.execute("insert into tb4 using stb2 tags(2,2) values(now, 1, 2, 3)"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 int, f3 int, t1 int, t2 int)"); + } + } + + @AfterClass + public static void afterClass() throws SQLException { + if (conn != null) { + conn.close(); + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java index 62bf704..813f613 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterTest.java @@ -1,31 +1,62 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; +import com.alibaba.datax.common.util.Configuration; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; +import java.util.List; public class TDengineWriterTest { + TDengineWriter.Job job; - @Test - public void testGetSchema() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", "wozai.fun", "6030", "test", "root", "taosdata"); - Connection conn = DriverManager.getConnection(jdbcUrl); - SchemaManager schemaManager = new SchemaManager(); - schemaManager.setStable("test1"); - schemaManager.getFromDB(conn); + @Before + public void before() { + job = new TDengineWriter.Job(); + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"batchSize\": \"1000\"" + + "}"); + job.setPluginJobConf(configuration); } @Test - public void dropTestTable() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = String.format("jdbc:TAOS://%s:%s/%s?user=%s&password=%s", "wozai.fun", "6030", "test", "root", "taosdata"); - Connection conn = DriverManager.getConnection(jdbcUrl); - Statement stmt = conn.createStatement(); - stmt.execute("drop table market_snapshot"); + public void jobInit() { + // when + job.init(); + + // assert + Configuration conf = job.getPluginJobConf(); + + Assert.assertEquals("root", conf.getString("username")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl")); + Assert.assertEquals(new Integer(1000), conf.getInt("batchSize")); + Assert.assertEquals("ts", conf.getString("column[0]")); + Assert.assertEquals("f2", conf.getString("column[2]")); } -} + + @Test + public void jobSplit() { + // when + job.init(); + List configurationList = job.split(10); + + // assert + Assert.assertEquals(10, configurationList.size()); + for (Configuration conf : configurationList) { + Assert.assertEquals("root", conf.getString("username")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); + Assert.assertEquals(new Integer(1000), conf.getInt("batchSize")); + Assert.assertEquals("ts", conf.getString("column[0]")); + Assert.assertEquals("f2", conf.getString("column[2]")); + + } + } + +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/mysql2tdengine.json b/tdenginewriter/src/test/resources/mysql2tdengine.json new file mode 100644 index 0000000..3848553 --- /dev/null +++ b/tdenginewriter/src/test/resources/mysql2tdengine.json @@ -0,0 +1,57 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "mysqlreader", + "parameter": { + "username": "root", + "password": "123456", + "column": [ + "*" + ], + "splitPk": "id", + "connection": [ + { + "table": [ + "t" + ], + "jdbcUrl": [ + "jdbc:mysql://192.168.56.105:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + ] + } + ] + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "t1" + ], + "connection": [ + { + "table": [ + "st" + ], + "jdbcUrl": "jdbc:TAOS://192.168.56.105:6030/test?timestampFormat=TIMESTAMP" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/opentsdb2tdengine.json b/tdenginewriter/src/test/resources/opentsdb2tdengine.json new file mode 100644 index 0000000..11264c9 --- /dev/null +++ b/tdenginewriter/src/test/resources/opentsdb2tdengine.json @@ -0,0 +1,36 @@ +{ + "job":{ + "content":[{ + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.56.105:4242", + "column": ["weather_temperature"], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "meters" + ], + "jdbcUrl": "jdbc:TAOS://192.168.56.105:6030/test?timestampFormat=TIMESTAMP" + } + ], + "batchSize": 1000 + } + } + }], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/tdengine2tdengine.json b/tdenginewriter/src/test/resources/tdengine2tdengine.json new file mode 100644 index 0000000..f0d0092 --- /dev/null +++ b/tdenginewriter/src/test/resources/tdengine2tdengine.json @@ -0,0 +1,51 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "meters" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "ts", + "current", + "voltage", + "phase", + "groupid", + "location" + ], + "beginDateTime": "2017-07-14 10:40:00", + "endDateTime": "2017-08-14 10:40:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "host": "192.168.56.105", + "port": 6030, + "dbName": "test2", + "username": "root", + "password": "taosdata", + "batchSize": 1000, + "stable": "meters" + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file From 67dd21edfee646ac6fbfae0d15b3a03fd6926ea4 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 16 Feb 2022 10:06:06 +0800 Subject: [PATCH 092/114] change --- tdenginewriter/pom.xml | 14 ++- .../writer/tdenginewriter/DataHandler.java | 1 - .../tdenginewriter/DefaultDataHandler.java | 54 ++++++++--- .../plugin/writer/tdenginewriter/Msg.java | 20 ---- .../writer/tdenginewriter/SchemaManager.java | 2 +- tdenginewriter/src/main/resources/plugin.json | 4 +- .../main/resources/plugin_job_template.json | 40 ++++---- .../resources/tdenginewritermsg.properties | 6 -- .../tdenginewritermsg_en_US.properties | 6 -- .../tdenginewritermsg_zh_CN.properties | 6 -- .../DefaultDataHandlerTest2.java | 2 + .../writer/tdenginewriter/EngineTest.java | 65 ------------- .../writer/tdenginewriter/MessageTest.java | 25 ----- .../tdenginewriter/Mysql2TDengineTest.java | 70 ++++++++++++++ .../tdenginewriter/Opentsdb2TDengineTest.java | 36 +++++++ .../tdenginewriter/Rdbms2TDengineTest.java | 4 + .../tdenginewriter/TDengine2TDengineTest.java | 68 ++++++++++++++ .../TDengine2TDengineTest3.java | 62 ++++++++++++ .../TDengine2TDengineTest4.java | 57 +++++++++++ .../{mysql2tdengine.json => m2t-1.json} | 27 ++++-- tdenginewriter/src/test/resources/t2t-1.json | 94 +++++++++++++++++++ tdenginewriter/src/test/resources/t2t-2.json | 92 ++++++++++++++++++ tdenginewriter/src/test/resources/t2t-3.json | 92 ++++++++++++++++++ tdenginewriter/src/test/resources/t2t-4.json | 72 ++++++++++++++ .../src/test/resources/tdengine2tdengine.json | 51 ---------- 25 files changed, 743 insertions(+), 227 deletions(-) delete mode 100644 tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java delete mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg.properties delete mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties delete mode 100644 tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java rename tdenginewriter/src/test/resources/{mysql2tdengine.json => m2t-1.json} (63%) create mode 100644 tdenginewriter/src/test/resources/t2t-1.json create mode 100644 tdenginewriter/src/test/resources/t2t-2.json create mode 100644 tdenginewriter/src/test/resources/t2t-3.json create mode 100644 tdenginewriter/src/test/resources/t2t-4.json delete mode 100644 tdenginewriter/src/test/resources/tdengine2tdengine.json diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 720ecbb..459ad0b 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -38,6 +38,12 @@ + + org.apache.commons + commons-lang3 + ${commons-lang3-version} + + com.alibaba.datax datax-common @@ -63,10 +69,12 @@ test - org.apache.commons - commons-lang3 - ${commons-lang3-version} + mysql + mysql-connector-java + 5.1.49 + test + diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java index 7afcb08..f22d4d6 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DataHandler.java @@ -4,6 +4,5 @@ import com.alibaba.datax.common.plugin.RecordReceiver; import com.alibaba.datax.common.plugin.TaskPluginCollector; public interface DataHandler { - int handle(RecordReceiver lineReceiver, TaskPluginCollector collector); } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 16be4c9..addda9c 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -14,10 +14,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.sql.*; -import java.util.ArrayList; +import java.util.*; import java.util.Date; -import java.util.List; -import java.util.Map; import java.util.stream.Collectors; public class DefaultDataHandler implements DataHandler { @@ -83,6 +81,11 @@ public class DefaultDataHandler implements DataHandler { } count++; } + + if (!recordBatch.isEmpty()) { + affectedRows = writeBatch(conn, recordBatch); + recordBatch.clear(); + } } catch (SQLException e) { throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); } @@ -302,15 +305,6 @@ public class DefaultDataHandler implements DataHandler { if (colMeta.type.equals("TIMESTAMP")) return dateAsLong(column) + "i64"; return "L'" + column.asString() + "'"; - case BYTES: - case STRING: - if (colMeta.type.equals("TIMESTAMP")) - return column.asString() + "i64"; - String value = column.asString(); - if (colMeta.type.startsWith("BINARY")) - return "'" + Utils.escapeSingleQuota(value) + "'"; - if (colMeta.type.startsWith("NCHAR")) - return "L'" + Utils.escapeSingleQuota(value) + "'"; case NULL: case BAD: return "NULL"; @@ -331,6 +325,16 @@ public class DefaultDataHandler implements DataHandler { if (colMeta.type.equals("BIGINT")) return column.asString() + "i64"; } + case BYTES: + case STRING: + if (colMeta.type.equals("TIMESTAMP")) + return column.asString() + "i64"; + String value = column.asString(); + value = value.replace("\"", "\\\""); + if (colMeta.type.startsWith("BINARY")) + return "\"" + value + "\""; + if (colMeta.type.startsWith("NCHAR")) + return "L\"" + value + "\""; case BOOL: default: return column.asString(); @@ -363,7 +367,9 @@ public class DefaultDataHandler implements DataHandler { boolean tagsAllMatch = columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { return colMeta.isTag; }).allMatch(colMeta -> { - return record.getColumn(indexOf(colMeta.field)).asString().equals(colMeta.value.toString()); + Column column = record.getColumn(indexOf(colMeta.field)); + boolean equals = equals(column, colMeta); + return equals; }); if (ignoreTagsUnmatched && !tagsAllMatch) @@ -386,6 +392,28 @@ public class DefaultDataHandler implements DataHandler { return executeUpdate(conn, sql); } + private boolean equals(Column column, ColumnMeta colMeta) { + switch (column.getType()) { + case BOOL: + return column.asBoolean().equals(Boolean.valueOf(colMeta.value.toString())); + case INT: + case LONG: + return column.asLong().equals(Long.valueOf(colMeta.value.toString())); + case DOUBLE: + return column.asDouble().equals(Double.valueOf(colMeta.value.toString())); + case NULL: + return colMeta.value == null; + case DATE: + return column.asDate().getTime() == ((Timestamp) colMeta.value).getTime(); + case BAD: + case BYTES: + return Arrays.equals(column.asBytes(), (byte[]) colMeta.value); + case STRING: + default: + return column.asString().equals(colMeta.value.toString()); + } + } + /** * table: ["weather"], column: ["ts, f1, f2, f3, t1, t2"] * sql: insert into weather (ts, f1, f2, f3, t1, t2) values( record[idx(ts), record[idx(f1)], ...) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java deleted file mode 100644 index 89730d3..0000000 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Msg.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import java.util.Locale; -import java.util.ResourceBundle; - -/** - * i18n message util - */ -public class Msg { - private static ResourceBundle bundle; - - static { - bundle = ResourceBundle.getBundle("tdenginewritermsg", Locale.getDefault()); - } - - public static String get(String key) { - return bundle.getString(key); - } - -} diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index 877b9b6..63d67a2 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -81,7 +81,7 @@ public class SchemaManager { for (String tbname : tables) { if (!tableMetas.containsKey(tbname)) { - LOG.error("table metadata of " + tbname + " is empty!"); + throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, "table metadata of " + tbname + " is empty!"); } } } catch (SQLException e) { diff --git a/tdenginewriter/src/main/resources/plugin.json b/tdenginewriter/src/main/resources/plugin.json index e54f65f..10d8e2c 100644 --- a/tdenginewriter/src/main/resources/plugin.json +++ b/tdenginewriter/src/main/resources/plugin.json @@ -3,7 +3,7 @@ "class": "com.alibaba.datax.plugin.writer.tdenginewriter.TDengineWriter", "description": { "useScene": "data migration to tdengine", - "mechanism": "use JNI or taos-jdbc to write data to tdengine." + "mechanism": "use taos-jdbcdriver to write data." }, - "developer": "zyyang-taosdata" + "developer": "support@taosdata.com" } \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/plugin_job_template.json b/tdenginewriter/src/main/resources/plugin_job_template.json index 39c9c96..3d30330 100644 --- a/tdenginewriter/src/main/resources/plugin_job_template.json +++ b/tdenginewriter/src/main/resources/plugin_job_template.json @@ -1,24 +1,20 @@ { - "name": "tdenginewriter", - "parameter": { - "host": "127.0.0.1", - "port": 6030, - "dbname": "test", - "user": "root", - "password": "taosdata", - "batchSize": 1000, - "stable": "weather", - "tagColumn": { - "station": 0 - }, - "fieldColumn": { - "latitude": 1, - "longtitude": 2, - "tmax": 4, - "tmin": 5 - }, - "timestampColumn":{ - "date": 3 - } - } + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "" + ], + "connection": [ + { + "table": [ + "" + ], + "jdbcUrl": "" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } } \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg.properties b/tdenginewriter/src/main/resources/tdenginewritermsg.properties deleted file mode 100644 index 48d862e..0000000 --- a/tdenginewriter/src/main/resources/tdenginewritermsg.properties +++ /dev/null @@ -1,6 +0,0 @@ -try_get_schema_from_db=fail to get structure info of target table from configure file and will try to get it from database -batch_size_too_small='batchSize' is too small, please increase it and try again -column_number_error=number of columns is less than expected -tag_value_error=tag columns include 'null' value -ts_value_error=timestamp column type error or null -infer_column_type_error=fail to infer column type: sample count %d, column index %d \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties deleted file mode 100644 index 48d862e..0000000 --- a/tdenginewriter/src/main/resources/tdenginewritermsg_en_US.properties +++ /dev/null @@ -1,6 +0,0 @@ -try_get_schema_from_db=fail to get structure info of target table from configure file and will try to get it from database -batch_size_too_small='batchSize' is too small, please increase it and try again -column_number_error=number of columns is less than expected -tag_value_error=tag columns include 'null' value -ts_value_error=timestamp column type error or null -infer_column_type_error=fail to infer column type: sample count %d, column index %d \ No newline at end of file diff --git a/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties b/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties deleted file mode 100644 index 38d2cbb..0000000 --- a/tdenginewriter/src/main/resources/tdenginewritermsg_zh_CN.properties +++ /dev/null @@ -1,6 +0,0 @@ -try_get_schema_from_db=\u65e0\u6cd5\u4ece\u914d\u7f6e\u6587\u4ef6\u83b7\u53d6\u8868\u7ed3\u6784\u4fe1\u606f\uff0c\u5c1d\u8bd5\u4ece\u6570\u636e\u5e93\u83b7\u53d6 -batch_size_too_small=batchSize\u592a\u5c0f\uff0c\u4f1a\u589e\u52a0\u81ea\u52a8\u7c7b\u578b\u63a8\u65ad\u9519\u8bef\u7684\u6982\u7387\uff0c\u5efa\u8bae\u6539\u5927\u540e\u91cd\u8bd5 -column_number_error=\u5b9e\u9645\u5217\u6570\u5c0f\u4e8e\u671f\u671b\u5217\u6570 -tag_value_error=\u6807\u7b7e\u5217\u5305\u542bnull -ts_value_error=\u65f6\u95f4\u6233\u5217\u4e3anull\u6216\u7c7b\u578b\u9519\u8bef -infer_column_type_error=\u6839\u636e\u91c7\u6837\u7684%d\u6761\u6570\u636e\uff0c\u65e0\u6cd5\u63a8\u65ad\u7b2c%d\u5217\u7684\u6570\u636e\u7c7b\u578b diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java index 48baad1..1ed32d6 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java @@ -27,6 +27,8 @@ public class DefaultDataHandlerTest2 { @Test public void writeSupTableBySchemaless() throws SQLException { + + // given Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java deleted file mode 100644 index 985c804..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/EngineTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import com.alibaba.datax.core.Engine; -import org.junit.Assert; -import org.junit.Test; - -import java.sql.*; - -public class EngineTest { - - @Test - public void opentsdb2tdengine() throws SQLException { - // when - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/opentsdb2tdengine.json"}; - System.setProperty("datax.home", "../target/datax/datax"); - try { - Engine.entry(params); - } catch (Throwable e) { - e.printStackTrace(); - } - - // assert - String jdbcUrl = "jdbc:TAOS://192.168.56.105:6030/test?timestampFormat=TIMESTAMP"; - try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("select count(*) from weather_temperature"); - int rows = 0; - while (rs.next()) { - rows = rs.getInt("count(*)"); - } - Assert.assertEquals(5, rows); - stmt.close(); - } - } - - @Test - public void mysql2tdengine() { - // given - // MYSQL SQL: - // create table t(id int primary key AUTO_INCREMENT, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double,ts timestamp, dt datetime,f7 nchar(64)) - // insert into t(f1,f2,f3,f4,f5,f6,ts,dt,f7) values(1,2,3,4,5,6,'2022-01-28 12:00:00','2022-01-28 12:00:00', 'beijing'); - // TDengine SQL: - // create table t(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double, dt timestamp,f7 nchar(64)); - - // when - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/mysql2tdengine.json"}; - System.setProperty("datax.home", "../target/datax/datax"); - try { - Engine.entry(params); - } catch (Throwable e) { - e.printStackTrace(); - } - } - - @Test - public void tdengine2tdengine() { - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/tdengine2tdengine.json"}; - System.setProperty("datax.home", "../target/datax/datax"); - try { - Engine.entry(params); - } catch (Throwable e) { - e.printStackTrace(); - } - } -} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java deleted file mode 100644 index b1b7ddd..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/MessageTest.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import org.junit.Test; - -import java.util.Locale; -import java.util.ResourceBundle; - -import org.junit.Assert; - -public class MessageTest { - @Test - public void testChineseMessage() { - Locale local = new Locale("zh", "CN"); - ResourceBundle bundle = ResourceBundle.getBundle("tdenginewritermsg", local); - String msg = bundle.getString("try_get_schema_fromdb"); - Assert.assertEquals("无法从配置文件获取表结构信息，尝试从数据库获取", msg); - } - - @Test - public void testDefaultMessage() { - ResourceBundle bundle = ResourceBundle.getBundle("tdenginewritermsg", Locale.getDefault()); - String msg = bundle.getString("try_get_schema_fromdb"); - System.out.println(msg); - } -} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java new file mode 100644 index 0000000..4a66271 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java @@ -0,0 +1,70 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.Random; + +public class Mysql2TDengineTest { + + private static final String host1 = "192.168.56.105"; + private static final String host2 = "192.168.1.93"; + private static final Random random = new Random(System.currentTimeMillis()); + + @Test + public void mysql2tdengine() throws Throwable { + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/m2t-1.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Before + public void before() throws SQLException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + String ts = sdf.format(new Date(System.currentTimeMillis())); + + final String url = "jdbc:mysql://" + host1 + ":3306/?useSSL=false&useUnicode=true&charset=UTF-8&generateSimpleParameterMetadata=true"; + try (Connection conn = DriverManager.getConnection(url, "root", "123456")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db1"); + stmt.execute("create database if not exists db1"); + stmt.execute("use db1"); + stmt.execute("create table stb1(id int primary key AUTO_INCREMENT, " + + "f1 tinyint, f2 smallint, f3 int, f4 bigint, " + + "f5 float, f6 double, " + + "ts timestamp, dt datetime," + + "f7 nchar(100), f8 varchar(100))"); + for (int i = 1; i <= 10; i++) { + String sql = "insert into stb1(f1, f2, f3, f4, f5, f6, ts, dt, f7, f8) values(" + + i + "," + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextInt(100) + "," + + random.nextFloat() + "," + random.nextDouble() + ", " + + "'" + ts + "', '" + ts + "', " + + "'中国北京朝阳望京abc', '中国北京朝阳望京adc')"; + stmt.execute(sql); + } + + stmt.close(); + } + + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041/"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2(" + + "ts timestamp, dt timestamp, " + + "f1 tinyint, f2 smallint, f3 int, f4 bigint, " + + "f5 float, f6 double, " + + "f7 nchar(100), f8 nchar(100))"); + + stmt.close(); + } + + } + +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java new file mode 100644 index 0000000..879eec2 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java @@ -0,0 +1,36 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.*; + +public class Opentsdb2TDengineTest { + + @Test + public void opentsdb2tdengine() throws SQLException { + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/opentsdb2tdengine.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + try { + Engine.entry(params); + } catch (Throwable e) { + e.printStackTrace(); + } + + // assert + String jdbcUrl = "jdbc:TAOS://192.168.56.105:6030/test?timestampFormat=TIMESTAMP"; + try (Connection conn = DriverManager.getConnection(jdbcUrl, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("select count(*) from weather_temperature"); + int rows = 0; + while (rs.next()) { + rows = rs.getInt("count(*)"); + } + Assert.assertEquals(5, rows); + stmt.close(); + } + } + +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java new file mode 100644 index 0000000..6a1170e --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java @@ -0,0 +1,4 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +public class Rdbms2TDengineTest { +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java new file mode 100644 index 0000000..0110be4 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java @@ -0,0 +1,68 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Random; + +public class TDengine2TDengineTest { + + private static final String host1 = "192.168.56.105"; + private static final String host2 = "192.168.1.93"; + private static final String db1 = "db1"; + private static final String stb1 = "stb1"; + private static final String db2 = "db2"; + private static final String stb2 = "stb2"; + private static Random random = new Random(System.currentTimeMillis()); + + @Test + public void case_01() throws Throwable { + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-1.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void case_02() throws Throwable { + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-2.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Before + public void before() throws SQLException { + final String url = "jdbc:TAOS-RS://" + host1 + ":6041"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists " + db1); + stmt.execute("create database if not exists " + db1); + stmt.execute("create table " + db1 + "." + stb1 + " (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + + " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); + for (int i = 0; i < 10; i++) { + String sql = "insert into " + db1 + ".t" + (i + 1) + " using " + db1 + "." + stb1 + " tags(now+" + i + "s," + + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextInt(100) + "," + + random.nextInt(100) + "," + random.nextFloat() + "," + random.nextDouble() + "," + + random.nextBoolean() + ",'abc123ABC','北京朝阳望京') values(now+" + i + "s, " + + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextInt(100) + "," + + random.nextInt(100) + "," + random.nextFloat() + "," + random.nextDouble() + "," + + random.nextBoolean() + ",'abc123ABC','北京朝阳望京')"; + stmt.execute(sql); + } + } + + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists " + db2); + stmt.execute("create database if not exists " + db2); + stmt.execute("create table " + db2 + "." + stb2 + " (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + + " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); + stmt.close(); + } + } +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java new file mode 100644 index 0000000..991f758 --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java @@ -0,0 +1,62 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.Random; + +public class TDengine2TDengineTest3 { + + private static final String host1 = "192.168.56.105"; + private static final String host2 = "192.168.1.93"; + private static Random random = new Random(System.currentTimeMillis()); + + @Test + public void case_03() throws Throwable { + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-3.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Before + public void before() throws SQLException { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + String ts = sdf.format(new Date(System.currentTimeMillis())); + + final String url = "jdbc:TAOS-RS://" + host1 + ":6041"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db1"); + stmt.execute("create database if not exists db1"); + stmt.execute("create table db1.stb1 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + + "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); + for (int i = 1; i <= 10; i++) { + String sql = "insert into db1.t" + i + " using db1.stb1 tags('" + ts + "'," + i + ",2,3,4,5.0,6.0,true,'abc123ABC','北京朝阳望京') " + + "values(now+" + i + "s, " + random.nextInt(100) + "," + random.nextInt(100) + "," + + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextFloat() + "," + + random.nextDouble() + "," + random.nextBoolean() + ",'abc123ABC','北京朝阳望京')"; + stmt.execute(sql); + } + } + + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + + " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); + + stmt.execute("create table db2.t1 using db2.stb2 tags('" + ts + "',1,2,3,4,5.0,6.0,true,'abc123ABC','北京朝阳望京')"); + stmt.close(); + } + + } +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java new file mode 100644 index 0000000..46ce0fd --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java @@ -0,0 +1,57 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.Random; + +public class TDengine2TDengineTest4 { + + private static final String host1 = "192.168.56.105"; + private static final String host2 = "192.168.1.93"; + private static Random random = new Random(System.currentTimeMillis()); + + @Test + public void case_04() throws Throwable { + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-4.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Before + public void before() throws SQLException { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + String ts = sdf.format(new Date(System.currentTimeMillis())); + + final String url = "jdbc:TAOS-RS://" + host1 + ":6041"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db1"); + stmt.execute("create database if not exists db1"); + stmt.execute("create table db1.weather (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + + "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100))"); + for (int i = 1; i <= 10; i++) { + String sql = "insert into db1.weather values(now+" + i + "s, " + random.nextInt(100) + "," + random.nextInt(100) + "," + + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextFloat() + "," + + random.nextDouble() + "," + random.nextBoolean() + ",'abc123ABC','北京朝阳望京')"; + stmt.execute(sql); + } + } + + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.weather (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + + "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100))"); + stmt.close(); + } + + } +} diff --git a/tdenginewriter/src/test/resources/mysql2tdengine.json b/tdenginewriter/src/test/resources/m2t-1.json similarity index 63% rename from tdenginewriter/src/test/resources/mysql2tdengine.json rename to tdenginewriter/src/test/resources/m2t-1.json index 3848553..dcacb4b 100644 --- a/tdenginewriter/src/test/resources/mysql2tdengine.json +++ b/tdenginewriter/src/test/resources/m2t-1.json @@ -8,16 +8,25 @@ "username": "root", "password": "123456", "column": [ - "*" + "ts", + "dt", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8" ], "splitPk": "id", "connection": [ { "table": [ - "t" + "stb1" ], "jdbcUrl": [ - "jdbc:mysql://192.168.56.105:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" + "jdbc:mysql://192.168.56.105:3306/db1?useSSL=false&useUnicode=true&characterEncoding=utf8" ] } ] @@ -30,16 +39,22 @@ "password": "taosdata", "column": [ "ts", + "dt", "f1", "f2", - "t1" + "f3", + "f4", + "f5", + "f6", + "f7", + "f8" ], "connection": [ { "table": [ - "st" + "stb2" ], - "jdbcUrl": "jdbc:TAOS://192.168.56.105:6030/test?timestampFormat=TIMESTAMP" + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" } ], "batchSize": 1000, diff --git a/tdenginewriter/src/test/resources/t2t-1.json b/tdenginewriter/src/test/resources/t2t-1.json new file mode 100644 index 0000000..5ca04d9 --- /dev/null +++ b/tdenginewriter/src/test/resources/t2t-1.json @@ -0,0 +1,94 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "tbname", + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10" + ], + "beginDateTime": "2022-02-15 00:00:00", + "endDateTime": "2022-02-16 00:00:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "tbname", + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2?timestampFormat=TIMESTAMP" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/t2t-2.json b/tdenginewriter/src/test/resources/t2t-2.json new file mode 100644 index 0000000..18130b6 --- /dev/null +++ b/tdenginewriter/src/test/resources/t2t-2.json @@ -0,0 +1,92 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10" + ], + "beginDateTime": "2022-02-15 00:00:00", + "endDateTime": "2022-02-16 00:00:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS://192.168.1.93:6030/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/t2t-3.json b/tdenginewriter/src/test/resources/t2t-3.json new file mode 100644 index 0000000..e0a2295 --- /dev/null +++ b/tdenginewriter/src/test/resources/t2t-3.json @@ -0,0 +1,92 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10" + ], + "beginDateTime": "2022-02-15 00:00:00", + "endDateTime": "2022-02-16 00:00:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10" + ], + "connection": [ + { + "table": [ + "t1" + ], + "jdbcUrl": "jdbc:TAOS://192.168.1.93:6030/db2?timestampFormat=TIMESTAMP" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/t2t-4.json b/tdenginewriter/src/test/resources/t2t-4.json new file mode 100644 index 0000000..9f1533b --- /dev/null +++ b/tdenginewriter/src/test/resources/t2t-4.json @@ -0,0 +1,72 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1?timestampFormat=TIMESTAMP" + } + ], + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9" + ], + "beginDateTime": "2022-02-15 00:00:00", + "endDateTime": "2022-02-16 00:00:00", + "splitInterval": "1d" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9" + ], + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/tdengine2tdengine.json b/tdenginewriter/src/test/resources/tdengine2tdengine.json deleted file mode 100644 index f0d0092..0000000 --- a/tdenginewriter/src/test/resources/tdengine2tdengine.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "tdenginereader", - "parameter": { - "username": "root", - "password": "taosdata", - "connection": [ - { - "table": [ - "meters" - ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" - } - ], - "column": [ - "ts", - "current", - "voltage", - "phase", - "groupid", - "location" - ], - "beginDateTime": "2017-07-14 10:40:00", - "endDateTime": "2017-08-14 10:40:00", - "splitInterval": "1d" - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "192.168.56.105", - "port": 6030, - "dbName": "test2", - "username": "root", - "password": "taosdata", - "batchSize": 1000, - "stable": "meters" - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file From 0680ce2e9708dc790f19e04a0cf4cf74e7e89e8f Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 18 Feb 2022 14:13:03 +0800 Subject: [PATCH 093/114] change --- tdenginewriter/pom.xml | 9 ++ .../tdenginewriter/DefaultDataHandler.java | 16 ++- .../tdenginewriter/DM2TDengineTest.java | 122 ++++++++++++++++++ .../DefaultDataHandlerTest.java | 82 ++++++++++-- .../DefaultDataHandlerTest2.java | 87 ------------- .../tdenginewriter/Opentsdb2TDengineTest.java | 2 +- .../tdenginewriter/Rdbms2TDengineTest.java | 4 - .../tdenginewriter/TDengine2TDengineTest.java | 99 +++++++++++--- .../TDengine2TDengineTest3.java | 62 --------- .../TDengine2TDengineTest4.java | 57 -------- .../src/test/resources/dm-schema.sql | 13 ++ tdenginewriter/src/test/resources/dm2t-1.json | 62 +++++++++ tdenginewriter/src/test/resources/dm2t-2.json | 62 +++++++++ tdenginewriter/src/test/resources/dm2t-3.json | 76 +++++++++++ tdenginewriter/src/test/resources/dm2t-4.json | 61 +++++++++ .../{opentsdb2tdengine.json => o2t-1.json} | 0 tdenginewriter/src/test/resources/t2t-4.json | 2 +- 17 files changed, 572 insertions(+), 244 deletions(-) create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java delete mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java create mode 100644 tdenginewriter/src/test/resources/dm-schema.sql create mode 100644 tdenginewriter/src/test/resources/dm2t-1.json create mode 100644 tdenginewriter/src/test/resources/dm2t-2.json create mode 100644 tdenginewriter/src/test/resources/dm2t-3.json create mode 100644 tdenginewriter/src/test/resources/dm2t-4.json rename tdenginewriter/src/test/resources/{opentsdb2tdengine.json => o2t-1.json} (100%) diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 459ad0b..9dd3aa0 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -74,6 +74,15 @@ 5.1.49 test + + + com.dameng + dm-jdbc + 1.8 + system + /Users/yangzy/IdeaProjects/DataX/tdenginewriter/src/test/resources/DmJdbcDriver18.jar + + diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index addda9c..4068be4 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -180,9 +180,21 @@ public class DefaultDataHandler implements DataHandler { private String buildColumnValue(ColumnMeta colMeta, Record record) { Column column = record.getColumn(indexOf(colMeta.field)); + TimestampPrecision timestampPrecision = schemaManager.loadDatabasePrecision(); switch (column.getType()) { - case DATE: - return "'" + column.asString() + "'"; + case DATE: { + Date value = column.asDate(); + switch (timestampPrecision) { + case MILLISEC: + return "" + (value.getTime()); + case MICROSEC: + return "" + (value.getTime() * 1000); + case NANOSEC: + return "" + (value.getTime() * 1000_000); + default: + return "'" + column.asString() + "'"; + } + } case BYTES: case STRING: if (colMeta.type.equals("TIMESTAMP")) diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java new file mode 100644 index 0000000..0eb91de --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java @@ -0,0 +1,122 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Random; + +public class DM2TDengineTest { + + private String host1 = "192.168.0.72"; + private String host2 = "192.168.1.93"; + private final Random random = new Random(System.currentTimeMillis()); + + @Test + public void dm2t_case01() throws Throwable { + // given + createSupTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/dm2t-1.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void dm2t_case02() throws Throwable { + // given + createSupAndSubTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/dm2t-2.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void dm2t_case03() throws Throwable { + // given + createTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/dm2t-3.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void dm2t_case04() throws Throwable { + // given + createSupTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/dm2t-4.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + private void createSupTable() throws SQLException { + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2(ts timestamp, f2 smallint, f4 bigint,f5 float, " + + "f6 double, f7 double, f8 bool, f9 nchar(100), f10 nchar(200)) tags(f1 tinyint,f3 int)"); + stmt.close(); + } + } + + private void createSupAndSubTable() throws SQLException { + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2(ts timestamp, f2 smallint, f4 bigint,f5 float, " + + "f6 double, f7 double, f8 bool, f9 nchar(100), f10 nchar(200)) tags(f1 tinyint,f3 int)"); + for (int i = 0; i < 10; i++) { + stmt.execute("create table db2.t" + (i + 1) + "_" + i + " using db2.stb2 tags(" + (i + 1) + "," + i + ")"); + } + stmt.close(); + } + } + + private void createTable() throws SQLException { + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint,f5 float, " + + "f6 double, f7 double, f8 bool, f9 nchar(100), f10 nchar(200))"); + stmt.close(); + } + } + + @Before + public void before() throws SQLException, ClassNotFoundException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + long ts = System.currentTimeMillis(); + + final String url = "jdbc:dm://" + host1 + ":5236"; + try (Connection conn = DriverManager.getConnection(url, "TESTUSER", "test123456")) { + conn.setAutoCommit(true); + Statement stmt = conn.createStatement(); + stmt.execute("drop table if exists stb1"); + stmt.execute("create table stb1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, " + + "f6 double, f7 NUMERIC(10,2), f8 BIT, f9 VARCHAR(100), f10 VARCHAR2(200))"); + for (int i = 0; i < 10; i++) { + String sql = "insert into stb1 values('" + sdf.format(new Date(ts + i * 1000)) + "'," + (i + 1) + "," + + random.nextInt(100) + "," + i + ",4,5.55,6.666,7.77," + (random.nextBoolean() ? 1 : 0) + + ",'abcABC123','北京朝阳望京DM')"; + stmt.execute(sql); + } + } + } + +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java index 733698a..3657a4f 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java @@ -22,12 +22,13 @@ import java.util.stream.IntStream; public class DefaultDataHandlerTest { - private static final String host = "192.168.56.105"; + private static final String host = "192.168.1.93"; private static Connection conn; @Test - public void writeSupTableBySQL() { + public void writeSupTableBySQL() throws SQLException { // given + createSupAndSubTable(); Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + @@ -55,6 +56,7 @@ public class DefaultDataHandlerTest { Map> columnMetas = schemaManager.loadColumnMetas(tables); handler.setTableMetas(tableMetas); handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); int count = handler.writeBatch(conn, recordList); @@ -63,8 +65,9 @@ public class DefaultDataHandlerTest { } @Test - public void writeSupTableBySQL_2() { + public void writeSupTableBySQL_2() throws SQLException { // given + createSupAndSubTable(); Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + @@ -91,6 +94,7 @@ public class DefaultDataHandlerTest { Map> columnMetas = schemaManager.loadColumnMetas(tables); handler.setTableMetas(tableMetas); handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); int count = handler.writeBatch(conn, recordList); @@ -99,8 +103,49 @@ public class DefaultDataHandlerTest { } @Test - public void writeSubTableWithTableName() { + public void writeSupTableBySchemaless() throws SQLException { // given + createSupTable(); + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"column\": [\"ts\", \"f1\", \"f2\", \"t1\"]," + + "\"table\":[\"stb1\"]," + + "\"jdbcUrl\":\"jdbc:TAOS://" + host + ":6030/scm_test\"," + + "\"batchSize\": \"1000\"" + + "}"); + String jdbcUrl = configuration.getString("jdbcUrl"); + Connection connection = DriverManager.getConnection(jdbcUrl, "root", "taosdata"); + long current = System.currentTimeMillis(); + List recordList = IntStream.range(1, 11).mapToObj(i -> { + Record record = new DefaultRecord(); + record.addColumn(new DateColumn(current + 1000 * i)); + record.addColumn(new LongColumn(1)); + record.addColumn(new LongColumn(2)); + record.addColumn(new StringColumn("t" + i + " 22")); + return record; + }).collect(Collectors.toList()); + + // when + DefaultDataHandler handler = new DefaultDataHandler(configuration); + List tables = configuration.getList("table", String.class); + SchemaManager schemaManager = new SchemaManager(connection); + Map tableMetas = schemaManager.loadTableMeta(tables); + Map> columnMetas = schemaManager.loadColumnMetas(tables); + handler.setTableMetas(tableMetas); + handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); + + int count = handler.writeBatch(connection, recordList); + + // then + Assert.assertEquals(10, count); + } + + @Test + public void writeSubTableWithTableName() throws SQLException { + // given + createSupAndSubTable(); Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + @@ -128,6 +173,7 @@ public class DefaultDataHandlerTest { Map> columnMetas = schemaManager.loadColumnMetas(tables); handler.setTableMetas(tableMetas); handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); int count = handler.writeBatch(conn, recordList); @@ -136,8 +182,9 @@ public class DefaultDataHandlerTest { } @Test - public void writeSubTableWithoutTableName() { + public void writeSubTableWithoutTableName() throws SQLException { // given + createSupAndSubTable(); Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + @@ -165,6 +212,7 @@ public class DefaultDataHandlerTest { Map> columnMetas = schemaManager.loadColumnMetas(tables); handler.setTableMetas(tableMetas); handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); int count = handler.writeBatch(conn, recordList); @@ -173,8 +221,9 @@ public class DefaultDataHandlerTest { } @Test - public void writeNormalTable() { + public void writeNormalTable() throws SQLException { // given + createSupAndSubTable(); Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + @@ -202,6 +251,7 @@ public class DefaultDataHandlerTest { Map> columnMetas = schemaManager.loadColumnMetas(tables); handler.setTableMetas(tableMetas); handler.setColumnMetas(columnMetas); + handler.setSchemaManager(schemaManager); int count = handler.writeBatch(conn, recordList); @@ -209,10 +259,8 @@ public class DefaultDataHandlerTest { Assert.assertEquals(10, count); } - @BeforeClass - public static void beforeClass() throws SQLException { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041", "root", "taosdata"); - try (Statement stmt = conn.createStatement()) { + private void createSupAndSubTable() throws SQLException { + try(Statement stmt = conn.createStatement()){ stmt.execute("drop database if exists scm_test"); stmt.execute("create database if not exists scm_test"); stmt.execute("use scm_test"); @@ -226,6 +274,20 @@ public class DefaultDataHandlerTest { } } + private void createSupTable() throws SQLException { + try (Statement stmt = conn.createStatement()){ + stmt.execute("drop database if exists scm_test"); + stmt.execute("create database if not exists scm_test"); + stmt.execute("use scm_test"); + stmt.execute("create table stb1(ts timestamp, f1 int, f2 int) tags(t1 nchar(32))"); + } + } + + @BeforeClass + public static void beforeClass() throws SQLException { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041", "root", "taosdata"); + } + @AfterClass public static void afterClass() throws SQLException { if (conn != null) { diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java deleted file mode 100644 index 1ed32d6..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest2.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import com.alibaba.datax.common.element.DateColumn; -import com.alibaba.datax.common.element.LongColumn; -import com.alibaba.datax.common.element.Record; -import com.alibaba.datax.common.element.StringColumn; -import com.alibaba.datax.common.util.Configuration; -import com.alibaba.datax.core.transport.record.DefaultRecord; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class DefaultDataHandlerTest2 { - - private static final String host = "192.168.1.93"; - private static Connection conn; - - @Test - public void writeSupTableBySchemaless() throws SQLException { - - - // given - Configuration configuration = Configuration.from("{" + - "\"username\": \"root\"," + - "\"password\": \"taosdata\"," + - "\"column\": [\"ts\", \"f1\", \"f2\", \"t1\"]," + - "\"table\":[\"stb1\"]," + - "\"jdbcUrl\":\"jdbc:TAOS://" + host + ":6030/scm_test\"," + - "\"batchSize\": \"1000\"" + - "}"); - - String jdbcUrl = configuration.getString("jdbcUrl"); - Connection connection = DriverManager.getConnection(jdbcUrl, "root", "taosdata"); - long current = System.currentTimeMillis(); - List recordList = IntStream.range(1, 11).mapToObj(i -> { - Record record = new DefaultRecord(); - record.addColumn(new DateColumn(current + 1000 * i)); - record.addColumn(new LongColumn(1)); - record.addColumn(new LongColumn(2)); - record.addColumn(new StringColumn("t" + i + " 22")); - return record; - }).collect(Collectors.toList()); - - // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); - List tables = configuration.getList("table", String.class); - SchemaManager schemaManager = new SchemaManager(connection); - Map tableMetas = schemaManager.loadTableMeta(tables); - Map> columnMetas = schemaManager.loadColumnMetas(tables); - handler.setTableMetas(tableMetas); - handler.setColumnMetas(columnMetas); - handler.setSchemaManager(schemaManager); - - int count = handler.writeBatch(connection, recordList); - - // then - Assert.assertEquals(10, count); - } - - @BeforeClass - public static void beforeClass() throws SQLException { - conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041", "root", "taosdata"); - try (Statement stmt = conn.createStatement()) { - stmt.execute("drop database if exists scm_test"); - stmt.execute("create database if not exists scm_test"); - stmt.execute("use scm_test"); - stmt.execute("create table stb1(ts timestamp, f1 int, f2 int) tags(t1 nchar(32))"); - } - } - - @AfterClass - public static void afterClass() throws SQLException { - if (conn != null) { - conn.close(); - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java index 879eec2..ad326f7 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java @@ -11,7 +11,7 @@ public class Opentsdb2TDengineTest { @Test public void opentsdb2tdengine() throws SQLException { // when - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/opentsdb2tdengine.json"}; + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/o2t-1.json"}; System.setProperty("datax.home", "../target/datax/datax"); try { Engine.entry(params); diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java deleted file mode 100644 index 6a1170e..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Rdbms2TDengineTest.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -public class Rdbms2TDengineTest { -} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java index 0110be4..9e95463 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java @@ -5,20 +5,21 @@ import org.junit.Before; import org.junit.Test; import java.sql.*; +import java.text.SimpleDateFormat; import java.util.Random; public class TDengine2TDengineTest { private static final String host1 = "192.168.56.105"; private static final String host2 = "192.168.1.93"; - private static final String db1 = "db1"; - private static final String stb1 = "stb1"; - private static final String db2 = "db2"; - private static final String stb2 = "stb2"; - private static Random random = new Random(System.currentTimeMillis()); + private static final Random random = new Random(System.currentTimeMillis()); @Test public void case_01() throws Throwable { + // given + createSupTable(); + + // when String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-1.json"}; System.setProperty("datax.home", "../target/datax/datax"); Engine.entry(params); @@ -26,24 +27,93 @@ public class TDengine2TDengineTest { @Test public void case_02() throws Throwable { + // given + createSupTable(); + + // when String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-2.json"}; System.setProperty("datax.home", "../target/datax/datax"); Engine.entry(params); } + @Test + public void case_03() throws Throwable { + // given + createSupAndSubTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-3.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void case_04() throws Throwable { + // given + createTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-4.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + private void createTable() throws SQLException { + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.weather (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + + "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100))"); + stmt.close(); + } + } + + private void createSupTable() throws SQLException { + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + + " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); + stmt.close(); + } + } + + private void createSupAndSubTable() throws SQLException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + final String ts = sdf.format(new Date(System.currentTimeMillis())); + + final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; + try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2"); + stmt.execute("create table db2.stb2 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + + " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); + + stmt.execute("create table db2.t1 using db2.stb2 tags('" + ts + "',1,2,3,4,5.0,6.0,true,'abc123ABC','北京朝阳望京')"); + stmt.close(); + } + } + @Before public void before() throws SQLException { final String url = "jdbc:TAOS-RS://" + host1 + ":6041"; try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { Statement stmt = conn.createStatement(); - stmt.execute("drop database if exists " + db1); - stmt.execute("create database if not exists " + db1); - stmt.execute("create table " + db1 + "." + stb1 + " (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + + stmt.execute("drop database if exists db1"); + stmt.execute("create database if not exists db1"); + stmt.execute("create table db1.stb1 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); for (int i = 0; i < 10; i++) { - String sql = "insert into " + db1 + ".t" + (i + 1) + " using " + db1 + "." + stb1 + " tags(now+" + i + "s," + + String sql = "insert into db1.t" + (i + 1) + " using db1.stb1 tags(now+" + i + "s," + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextFloat() + "," + random.nextDouble() + "," + random.nextBoolean() + ",'abc123ABC','北京朝阳望京') values(now+" + i + "s, " + @@ -53,16 +123,5 @@ public class TDengine2TDengineTest { stmt.execute(sql); } } - - final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; - try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { - Statement stmt = conn.createStatement(); - stmt.execute("drop database if exists " + db2); - stmt.execute("create database if not exists " + db2); - stmt.execute("create table " + db2 + "." + stb2 + " (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + - " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + - "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); - stmt.close(); - } } } diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java deleted file mode 100644 index 991f758..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest3.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import com.alibaba.datax.core.Engine; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.text.SimpleDateFormat; -import java.util.Random; - -public class TDengine2TDengineTest3 { - - private static final String host1 = "192.168.56.105"; - private static final String host2 = "192.168.1.93"; - private static Random random = new Random(System.currentTimeMillis()); - - @Test - public void case_03() throws Throwable { - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-3.json"}; - System.setProperty("datax.home", "../target/datax/datax"); - Engine.entry(params); - } - - @Before - public void before() throws SQLException { - - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - String ts = sdf.format(new Date(System.currentTimeMillis())); - - final String url = "jdbc:TAOS-RS://" + host1 + ":6041"; - try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { - Statement stmt = conn.createStatement(); - - stmt.execute("drop database if exists db1"); - stmt.execute("create database if not exists db1"); - stmt.execute("create table db1.stb1 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + - "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + - "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); - for (int i = 1; i <= 10; i++) { - String sql = "insert into db1.t" + i + " using db1.stb1 tags('" + ts + "'," + i + ",2,3,4,5.0,6.0,true,'abc123ABC','北京朝阳望京') " + - "values(now+" + i + "s, " + random.nextInt(100) + "," + random.nextInt(100) + "," + - random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextFloat() + "," + - random.nextDouble() + "," + random.nextBoolean() + ",'abc123ABC','北京朝阳望京')"; - stmt.execute(sql); - } - } - - final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; - try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { - Statement stmt = conn.createStatement(); - stmt.execute("drop database if exists db2"); - stmt.execute("create database if not exists db2"); - stmt.execute("create table db2.stb2 (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint," + - " f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, " + - "t3 smallint, t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(1000))"); - - stmt.execute("create table db2.t1 using db2.stb2 tags('" + ts + "',1,2,3,4,5.0,6.0,true,'abc123ABC','北京朝阳望京')"); - stmt.close(); - } - - } -} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java deleted file mode 100644 index 46ce0fd..0000000 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest4.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.alibaba.datax.plugin.writer.tdenginewriter; - -import com.alibaba.datax.core.Engine; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.text.SimpleDateFormat; -import java.util.Random; - -public class TDengine2TDengineTest4 { - - private static final String host1 = "192.168.56.105"; - private static final String host2 = "192.168.1.93"; - private static Random random = new Random(System.currentTimeMillis()); - - @Test - public void case_04() throws Throwable { - String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2t-4.json"}; - System.setProperty("datax.home", "../target/datax/datax"); - Engine.entry(params); - } - - @Before - public void before() throws SQLException { - - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - String ts = sdf.format(new Date(System.currentTimeMillis())); - - final String url = "jdbc:TAOS-RS://" + host1 + ":6041"; - try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { - Statement stmt = conn.createStatement(); - - stmt.execute("drop database if exists db1"); - stmt.execute("create database if not exists db1"); - stmt.execute("create table db1.weather (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + - "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100))"); - for (int i = 1; i <= 10; i++) { - String sql = "insert into db1.weather values(now+" + i + "s, " + random.nextInt(100) + "," + random.nextInt(100) + "," + - random.nextInt(100) + "," + random.nextInt(100) + "," + random.nextFloat() + "," + - random.nextDouble() + "," + random.nextBoolean() + ",'abc123ABC','北京朝阳望京')"; - stmt.execute(sql); - } - } - - final String url2 = "jdbc:TAOS-RS://" + host2 + ":6041"; - try (Connection conn = DriverManager.getConnection(url2, "root", "taosdata")) { - Statement stmt = conn.createStatement(); - stmt.execute("drop database if exists db2"); - stmt.execute("create database if not exists db2"); - stmt.execute("create table db2.weather (ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, " + - "f5 float, f6 double, f7 bool, f8 binary(100), f9 nchar(100))"); - stmt.close(); - } - - } -} diff --git a/tdenginewriter/src/test/resources/dm-schema.sql b/tdenginewriter/src/test/resources/dm-schema.sql new file mode 100644 index 0000000..fcac83e --- /dev/null +++ b/tdenginewriter/src/test/resources/dm-schema.sql @@ -0,0 +1,13 @@ +select tablespace_name from dba_data_files; + +create tablespace test datafile '/home/dmdba/dmdbms/data/DAMENG/test.dbf' size 32 autoextend on next 1 maxsize 1024; + +create user TESTUSER identified by test123456 default tablespace test; + +grant dba to TESTUSER; + +select * from user_tables; + +drop table if exists stb1; + +create table stb1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double, f7 NUMERIC(10,2), f8 BIT, f9 VARCHAR(100), f10 VARCHAR2(200)); diff --git a/tdenginewriter/src/test/resources/dm2t-1.json b/tdenginewriter/src/test/resources/dm2t-1.json new file mode 100644 index 0000000..183786b --- /dev/null +++ b/tdenginewriter/src/test/resources/dm2t-1.json @@ -0,0 +1,62 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "rdbmsreader", + "parameter": { + "username": "TESTUSER", + "password": "test123456", + "connection": [ + { + "querySql": [ + "select concat(concat(concat('t', f1), '_'),f3) as tbname,* from stb1;" + ], + "jdbcUrl": [ + "jdbc:dm://192.168.0.72:5236" + ] + } + ], + "fetchSize": 1024 + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "tbname", + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "f10" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/dm2t-2.json b/tdenginewriter/src/test/resources/dm2t-2.json new file mode 100644 index 0000000..dfea82b --- /dev/null +++ b/tdenginewriter/src/test/resources/dm2t-2.json @@ -0,0 +1,62 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "rdbmsreader", + "parameter": { + "username": "TESTUSER", + "password": "test123456", + "connection": [ + { + "querySql": [ + "select concat(concat(concat('t', f1), '_'),f3) as tbname,* from stb1;" + ], + "jdbcUrl": [ + "jdbc:dm://192.168.0.72:5236" + ] + } + ], + "fetchSize": 1024, + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "tbname", + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "f10" + ], + "connection": [ + { + "table": [ + "t1_0" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/dm2t-3.json b/tdenginewriter/src/test/resources/dm2t-3.json new file mode 100644 index 0000000..cd96a53 --- /dev/null +++ b/tdenginewriter/src/test/resources/dm2t-3.json @@ -0,0 +1,76 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "rdbmsreader", + "parameter": { + "username": "TESTUSER", + "password": "test123456", + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "f10" + ], + "splitPk": "f1", + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": [ + "jdbc:dm://192.168.0.72:5236" + ] + } + ], + "fetchSize": 1024, + "where": "1 = 1" + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "f10" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/dm2t-4.json b/tdenginewriter/src/test/resources/dm2t-4.json new file mode 100644 index 0000000..5f169d5 --- /dev/null +++ b/tdenginewriter/src/test/resources/dm2t-4.json @@ -0,0 +1,61 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "rdbmsreader", + "parameter": { + "username": "TESTUSER", + "password": "test123456", + "connection": [ + { + "querySql": [ + "select * from stb1" + ], + "jdbcUrl": [ + "jdbc:dm://192.168.0.72:5236" + ] + } + ], + "fetchSize": 1024 + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "f10" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS://192.168.1.93:6030/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/opentsdb2tdengine.json b/tdenginewriter/src/test/resources/o2t-1.json similarity index 100% rename from tdenginewriter/src/test/resources/opentsdb2tdengine.json rename to tdenginewriter/src/test/resources/o2t-1.json diff --git a/tdenginewriter/src/test/resources/t2t-4.json b/tdenginewriter/src/test/resources/t2t-4.json index 9f1533b..b771636 100644 --- a/tdenginewriter/src/test/resources/t2t-4.json +++ b/tdenginewriter/src/test/resources/t2t-4.json @@ -10,7 +10,7 @@ "connection": [ { "table": [ - "weather" + "stb1" ], "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1?timestampFormat=TIMESTAMP" } From c2a76420cc07fa20f190b118570f99477eed830f Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 18 Feb 2022 14:16:22 +0800 Subject: [PATCH 094/114] change --- tdenginewriter/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 9dd3aa0..9ce6899 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -80,7 +80,7 @@ dm-jdbc 1.8 system - /Users/yangzy/IdeaProjects/DataX/tdenginewriter/src/test/resources/DmJdbcDriver18.jar + ${project.basedir}/src/test/resources/DmJdbcDriver18.jar From e3f9e1219e5f01de7ed234bc5108ce25b5133848 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 18 Feb 2022 18:34:34 +0800 Subject: [PATCH 095/114] change fastjson version --- tdenginewriter/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 9ce6899..97c750d 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -23,7 +23,7 @@ com.alibaba fastjson - 1.2.58 + 1.2.78 From fad2760ca2fd3dc9ff57e85d02c7d7f6b87f78d0 Mon Sep 17 00:00:00 2001 From: zyyang Date: Sun, 20 Feb 2022 11:27:37 +0800 Subject: [PATCH 096/114] add s2t test case --- .../tdenginewriter/Stream2TDengineTest.java | 69 ++++++ .../src/test/resources/defaultJob.json | 211 ++++++++++++++++++ .../src/test/resources/dm-schema.sql | 27 ++- 3 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java create mode 100644 tdenginewriter/src/test/resources/defaultJob.json diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java new file mode 100644 index 0000000..cfd77fd --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java @@ -0,0 +1,69 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +public class Stream2TDengineTest { + + private String host2 = "192.168.56.105"; + private String precision; + + @Test + public void s2t_case1() throws Throwable { + // given + createSupTable("ms"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/defaultJob.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void s2t_case2() throws Throwable { + // given + createSupTable("us"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/defaultJob.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void s2t_case3() throws Throwable { + // given + createSupTable("ns"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/defaultJob.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + void createSupTable(String precision) throws SQLException { + + final String url = "jdbc:TAOS-RS://" + host2 + ":6041/"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db2"); + stmt.execute("create database if not exists db2 precision '" + precision + "'"); + stmt.execute("create table db2.stb2(ts1 timestamp, ts2 timestamp,ts3 timestamp,ts4 timestamp,ts5 timestamp," + + "ts6 timestamp,ts7 timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double," + + "f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp,t2 timestamp,t3 timestamp,t4 timestamp," + + "t5 timestamp,t6 timestamp,t7 timestamp, t8 tinyint, t9 smallint, t10 int, t11 bigint, t12 float," + + "t13 double, t14 bool, t15 binary(100), t16 nchar(100))"); + + stmt.close(); + } + + } + +} diff --git a/tdenginewriter/src/test/resources/defaultJob.json b/tdenginewriter/src/test/resources/defaultJob.json new file mode 100644 index 0000000..2548fc9 --- /dev/null +++ b/tdenginewriter/src/test/resources/defaultJob.json @@ -0,0 +1,211 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "streamreader", + "parameter": { + "column": [ + { + "type": "string", + "value": "tb1" + }, + { + "type": "date", + "value": "2022-02-20 12:00:01" + }, + { + "type": "date", + "value": "2022-02-20 12:00:02.123", + "dateFormat": "yyyy-MM-dd HH:mm:ss.SSS" + }, + { + "type": "date", + "value": "2022-02-20 12:00:03.123456", + "dateFormat": "yyyy-MM-dd HH:mm:ss.SSSSSS" + }, + { + "type": "date", + "value": "2022-02-20 12:00:04.123456789", + "dateFormat": "yyyy-MM-dd HH:mm:ss.SSSSSSSSS" + }, + { + "type": "string", + "value": "2022-02-20 12:00:05.123" + }, + { + "type": "string", + "value": "2022-02-20 12:00:06.123456" + }, + { + "type": "string", + "value": "2022-02-20 12:00:07.123456789" + }, + { + "type": "long", + "random": "0, 10" + }, + { + "type": "long", + "random": "0, 100" + }, + { + "type": "long", + "random": "0, 1000" + }, + { + "type": "long", + "random": "0, 10000" + }, + { + "type": "double", + "random": "0, 10" + }, + { + "type": "double", + "random": "10, 20" + }, + { + "type": "bool", + "random": "0, 50" + }, + { + "type": "bytes", + "random": "0, 10" + }, + { + "type": "string", + "random": "0, 10" + }, + { + "type": "date", + "value": "2022-02-20 12:00:01" + }, + { + "type": "date", + "value": "2022-02-20 12:00:02.123", + "dateFormat": "yyyy-MM-dd HH:mm:ss.SSS" + }, + { + "type": "date", + "value": "2022-02-20 12:00:03.123456", + "dateFormat": "yyyy-MM-dd HH:mm:ss.SSSSSS" + }, + { + "type": "date", + "value": "2022-02-20 12:00:04.123456789", + "dateFormat": "yyyy-MM-dd HH:mm:ss.SSSSSSSSS" + }, + { + "type": "string", + "value": "2022-02-20 12:00:05.123" + }, + { + "type": "string", + "value": "2022-02-20 12:00:06.123456" + }, + { + "type": "string", + "value": "2022-02-20 12:00:07.123456789" + }, + { + "type": "long", + "value": 1 + }, + { + "type": "long", + "value": 2 + }, + { + "type": "long", + "value": 3 + }, + { + "type": "long", + "value": 4 + }, + { + "type": "double", + "value": 5.55 + }, + { + "type": "double", + "value": 6.666666 + }, + { + "type": "bool", + "value": true + }, + { + "type": "bytes", + "value": "abcABC123" + }, + { + "type": "string", + "value": "北京朝阳望京" + } + ], + "sliceRecordCount": 10 + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "tbname", + "ts1", + "ts2", + "ts3", + "ts4", + "ts5", + "ts6", + "ts7", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "t1", + "t2", + "t3", + "t4", + "t5", + "t6", + "t7", + "t8", + "t9", + "t10", + "t11", + "t12", + "t13", + "t14", + "t15", + "t16" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db2" + } + ], + "batchSize": 100, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/dm-schema.sql b/tdenginewriter/src/test/resources/dm-schema.sql index fcac83e..d9985b1 100644 --- a/tdenginewriter/src/test/resources/dm-schema.sql +++ b/tdenginewriter/src/test/resources/dm-schema.sql @@ -1,13 +1,30 @@ -select tablespace_name from dba_data_files; +select tablespace_name +from dba_data_files; -create tablespace test datafile '/home/dmdba/dmdbms/data/DAMENG/test.dbf' size 32 autoextend on next 1 maxsize 1024; +create +tablespace test datafile '/home/dmdba/dmdbms/data/DAMENG/test.dbf' size 32 autoextend on next 1 maxsize 1024; -create user TESTUSER identified by test123456 default tablespace test; +create +user TESTUSER identified by test123456 default tablespace test; grant dba to TESTUSER; -select * from user_tables; +select * +from user_tables; drop table if exists stb1; -create table stb1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double, f7 NUMERIC(10,2), f8 BIT, f9 VARCHAR(100), f10 VARCHAR2(200)); +create table stb1 +( + ts timestamp, + f1 tinyint, + f2 smallint, + f3 int, + f4 bigint, + f5 float, + f6 double, + f7 NUMERIC(10, 2), + f8 BIT, + f9 VARCHAR(100), + f10 VARCHAR2(200) +); From c96256c9a270992293eba61e95379ee2df4201cd Mon Sep 17 00:00:00 2001 From: zyyang Date: Sun, 20 Feb 2022 15:20:19 +0800 Subject: [PATCH 097/114] change doc-CN --- tdenginewriter/doc/tdenginewriter-CN.md | 433 ++++++++---------------- 1 file changed, 141 insertions(+), 292 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md index a868f8f..910425c 100644 --- a/tdenginewriter/doc/tdenginewriter-CN.md +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -4,21 +4,40 @@ ## 1 快速介绍 -TDengineWriter插件实现了写入数据到TDengine数据库功能。可用于离线同步其它数据库的数据到TDengine。 +TDengineWriter插件实现了写入数据到TDengine数据库目标表的功能。底层实现上，TDengineWriter通过JDBC连接TDengine，按照TDengine的SQL语法，执行insert语句/schemaless语句，将数据写入TDengine。 + +TDengineWriter可以作为数据迁移工具供DBA将其它数据库的数据导入到TDengine。 + + ## 2 实现原理 -TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据reader的类型解析数据。目前有两种写入方式： +TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，通过JDBC Driver连接TDengine，执行insert语句/schemaless语句，将数据写入TDengine。 -1. 对于OpenTSDBReader, TDengineWriter通过JNI方式调用TDengine客户端库文件（taos.lib或taos.dll）中的方法，使用[schemaless的方式](https://www.taosdata.com/cn/documentation/insert#schemaless)写入。 +在TDengine中，table可以分成超级表、子表、普通表三种类型，超级表和子表包括colum和tag，子表的tag列的值为固定值，普通表与关系型数据库中表的概念一致。（详细请参考：[数据模型](https://www.taosdata.com/docs/cn/v2.0/architecture#model) ） + +TDengineWriter支持向超级表、子表、普通表中写入数据，按照table的类型和column参数中是否包含tbname，使用以下方法进行写入： + +1. table为超级表，column中指定tbname：使用自动建表的insert语句，使用tbname作为子表的名称。 +2. table为超级表，column中未指定tbname：使用schemaless写入，TDengine会根据超级表名、tag值计算一个子表名称。 +3. table为子表：使用insert语句写入，ignoreTagUnmatched参数为true时，忽略record中tag值与table的tag值不一致的数据。 +4. table为普通表：使用insert语句写入。 -2. 对于其它数据源,会根据配置生成SQL语句, 通过[taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java)批量写入。 -这样区分的原因是OpenTSDBReader将opentsdb的数据统一读取为json字符串，Writer端接收到的数据只有1列。而其它Reader插件一般会把数据放在不同列。 ## 3 功能说明 -### 3.1 从OpenTSDB到TDengine -#### 3.1.1 配置样例 +### 3.1 配置样例 + +配置一个写入TDengine的作业 + +先在TDengine上创建超级表： + +```sql +create database if not exists test; +create table test.weather (ts timestamp, temperature int, humidity double) tags(is_normal bool, device_id binary(100), address nchar(100)); +``` + +使用下面的Job配置，将数据写入TDengine： ```json { @@ -26,286 +45,65 @@ TDengineWriter 通过 DataX 框架获取 Reader生成的协议数据，根据rea "content": [ { "reader": { - "name": "opentsdbreader", + "name": "streamreader", "parameter": { - "endpoint": "http://192.168.1.180:4242", "column": [ - "weather_temperature" + { + "type": "string", + "value": "tb1" + }, + { + "type": "date", + "value": "2022-02-20 12:00:01" + }, + { + "type": "long", + "random": "0, 10" + }, + { + "type": "double", + "random": "0, 10" + }, + { + "type": "bool", + "random": "0, 50" + }, + { + "type": "bytes", + "value": "abcABC123" + }, + { + "type": "string", + "value": "北京朝阳望京" + } ], - "beginDateTime": "2021-01-01 00:00:00", - "endDateTime": "2021-01-01 01:00:00" + "sliceRecordCount": 1 } }, "writer": { "name": "tdenginewriter", "parameter": { - "host": "192.168.1.180", - "port": 6030, - "dbName": "test", "username": "root", - "password": "taosdata" - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} -``` - -#### 3.1.2 参数说明 - -| 参数 | 描述 | 是否必选 | 默认值 | -| --------- | -------------------- | -------- | -------- | -| host | TDengine实例的host | 是 | 无 | -| port | TDengine实例的port | 是 | 无 | -| username | TDengine实例的用户名 | 否 | root | -| password | TDengine实例的密码 | 否 | taosdata | -| dbName | 目的数据库的名称 | 是 | 无 | -| batchSize | 每次批量插入多少记录 | 否 | 1 | - - -#### 3.1.3 类型转换 - -目前，由于OpenTSDBReader将opentsdb的数据统一读取为json字符串，TDengineWriter 在做Opentsdb到TDengine的迁移时，按照以下类型进行处理： - -| OpenTSDB数据类型 | DataX 内部类型 | TDengine 数据类型 | -| ---------------- | -------------- | ----------------- | -| timestamp | Date | timestamp | -| Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | -| Integer（tag） | String | binary | -| Float（tag） | String | binary | -| String（tag） | String | binary | - -### 3.2 从MongoDB到TDengine - -#### 3.2.1 配置样例 -```json -{ - "job": { - "setting": { - "speed": { - "channel": 2 - } - }, - "content": [ - { - "reader": { - "name": "mongodbreader", - "parameter": { - "address": [ - "127.0.0.1:27017" - ], - "userName": "user", - "mechanism": "SCRAM-SHA-1", - "userPassword": "password", - "authDb": "admin", - "dbName": "test", - "collectionName": "stock", - "column": [ - { - "name": "stockID", - "type": "string" - }, - { - "name": "tradeTime", - "type": "date" - }, - { - "name": "lastPrice", - "type": "double" - }, - { - "name": "askPrice1", - "type": "double" - }, - { - "name": "bidPrice1", - "type": "double" - }, - { - "name": "volume", - "type": "int" - } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "localhost", - "port": 6030, - "dbName": "test", - "username": "root", - "password": "taosdata", - "stable": "stock", - "tagColumn": { - "industry": "energy", - "stockID": 0 - }, - "fieldColumn": { - "lastPrice": 2, - "askPrice1": 3, - "bidPrice1": 4, - "volume": 5 - }, - "timestampColumn": { - "tradeTime": 1 - } - } - } - } - ] - } -} -``` - -**注：本配置的writer部分同样适用于关系型数据库** - - -#### 3.2.2 参数说明 -| 参数 | 描述 | 是否必选 | 默认值 | 备注 | -| --------------- | -------------------- | ---------------- | -------- | ------------------ | -| host | TDengine实例的host | 是 | 无 | -| port | TDengine实例的port | 是 | 无 | -| username | TDengine实例的用户名 | 否 | root | -| password | TDengine实例的密码 | 否 | taosdata | -| dbName | 目的数据库的名称 | 是 | 无 | -| batchSize | 每次批量插入多少记录 | 否 | 1000 | -| stable | 目标超级表的名称 | 是(OpenTSDB除外) | 无 | -| tagColumn | 格式:{tagName1: tagInd1, tagName2: tagInd2}, 标签列在写插件收到的Record中的位置和列名 | 否 | 无 | 位置索引均从0开始, tagInd如果为字符串, 表示固定标签值，不需要从源数据中获取 | -| fieldColumn | 格式:{fdName1: fdInd1, fdName2: fdInd2}, 字段列在写插件收到的Record中的位置和列名 | 否 | 无 | | -| timestampColumn | 格式:{tsColName: tsColIndex}, 时间戳列在写插件收到的Record中的位置和列名 | 否 | 无 | 时间戳列只能有一个 | - -示例配置中tagColumn有一个industry，它的值是一个固定的字符串“energy”, 作用是给导入的所有数据加一个值为"energy"的固定标签industry。这个应用场景可以是：在源库中，有多个设备采集的数据分表存储，设备名就是表名，可以用这个机制把设备名称转化为标签。 - -#### 3.2.3 自动建表规则 -##### 3.2.3.1 超级表创建规则 - -如果配置了tagColumn、 fieldColumn和timestampColumn将会在插入第一条数据前，自动创建超级表。
-数据列的类型从第1条记录自动推断, 标签列默认类型为`NCHAR(64)`, 比如示例配置，可能生成以下建表语句： - -```sql -CREATE STABLE IF NOT EXISTS market_snapshot ( - tadetime TIMESTAMP, - lastprice DOUBLE, - askprice1 DOUBLE, - bidprice1 DOUBLE, - volume INT -) -TAGS( - industry NCHAR(64), - stockID NCHAR(64) -); -``` - -##### 3.2.3.2 子表创建规则 - -子表结构与超级表相同，子表表名生成规则： -1. 将标签的value 组合成为如下的字符串: `tag_value1!tag_value2!tag_value3`。 -2. 计算该字符串的 MD5 散列值 "md5_val"。 -3. "t_md5val"作为子表名。其中的 "t" 是固定的前缀。 - -#### 3.2.4 用户提前建表 - -如果你已经创建好目标超级表，那么tagColumn、 fieldColumn和timestampColumn三个字段均可省略, 插件将通过执行通过`describe stableName`获取表结构的信息。 -此时要求接收到的Record中Column的顺序和执行`describe stableName`返回的列顺序相同， 比如通过`describe stableName`返回以下内容： -``` - Field | Type | Length | Note | -================================================================================= - ts | TIMESTAMP | 8 | | - current | DOUBLE | 8 | | - location | BINARY | 10 | TAG | -``` -那么插件收到的数据第1列必须代表时间戳，第2列必须代表电流，第3列必须代表位置。 - -#### 3.2.5 注意事项 - -1. tagColumn、 fieldColumn和timestampColumn三个字段用于描述目标表的结构信息，这三个配置字段必须同时存在或同时省略。 -2. 如果存在以上三个配置，且目标表也已经存在，则两者必须一致。**一致性**由用户自己保证，插件不做检查。不一致可能会导致插入失败或插入数据错乱。 - -#### 3.2.6 类型转换 - -| MongoDB 数据类型 | DataX 内部类型 | TDengine 数据类型 | -| ---------------- | -------------- | ----------------- | -| int, Long | Long | BIGINT | -| double | Double | DOUBLE | -| string, array | String | NCHAR(64) | -| date | Date | TIMESTAMP | -| boolean | Boolean | BOOL | -| bytes | Bytes | BINARY(64) | - -### 3.3 从关系型数据库到TDengine -writer部分的配置规则和上述MongoDB的示例是一样的，这里给出一个MySQL的示例。 - -#### 3.3.1 MySQL中表结构 -```sql -CREATE TABLE IF NOT EXISTS weather( - station varchar(100), - latitude DOUBLE, - longtitude DOUBLE, - `date` DATE, - TMAX int, - TMIN int -) -``` - -#### 3.3.2 配置文件示例 - -```json -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "username": "root", - "password": "passw0rd", + "password": "taosdata", "column": [ - "*" + "tbname", + "ts", + "temperature", + "humidity", + "is_normal", + "device_id", + "address" ], - "splitPk": "station", "connection": [ { "table": [ "weather" ], - "jdbcUrl": [ - "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" - ] + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test" } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "127.0.0.1", - "port": 6030, - "dbName": "test", - "username": "root", - "password": "taosdata", - "batchSize": 1000, - "stable": "weather", - "tagColumn": { - "station": 0 - }, - "fieldColumn": { - "latitude": 1, - "longtitude": 2, - "tmax": 4, - "tmin": 5 - }, - "timestampColumn":{ - "date": 3 - } + ], + "batchSize": 100, + "ignoreTagsUnmatched": true } } } @@ -319,6 +117,72 @@ CREATE TABLE IF NOT EXISTS weather( } ``` +### 3.2 参数说明 + +* jdbcUrl + * 描述：数据源的JDBC连接信息，TDengine的JDBC信息请参考：[Java连接器的使用](https://www.taosdata.com/docs/cn/v2.0/connector/java#url) + * 必选：是 + * 默认值：无 +* username + * 描述：用户名 + * 必选：是 + * 默认值：无 +* password + * 描述：用户名的密码 + * 必选：是 + * 默认值：无 +* table + * 描述：表名的集合，table应该包含column参数中的所有列（tbname除外）。注意，column中的tbname会被当作TDengine中子表名使用。 + * 必选：是 + * 默认值：无 +* column + * 描述：字段名的集合，字段的顺序应该与record中column的 + * 必选：是 + * 默认值：无 +* batchSize + * 描述： + * 必选：否 + * 默认值：1 +* ignoreTagsUnmatched + * 描述：当table为TDengine中的一张子表，table具有tag值。如果数据的tag值与table的tag值不想等，数据不写入到table中。 + * 必选：否 + * 默认值：false + + +### 3.3 类型转换 + +datax中的数据类型，可以映射到TDengine的数据类型 + +| DataX 内部类型 | TDengine 数据类型 | +| -------------- | ----------------------------------------- | +| INT | TINYINT, SMALLINT, INT | +| LONG | TIMESTAMP, TINYINT, SMALLINT, INT, BIGINT | +| DOUBLE | FLOAT, DOUBLE | +| STRING | TIMESTAMP, BINARY, NCHAR | +| BOOL | BOOL | +| DATE | TIMESTAMP | +| BYTES | BINARY | + + + +### 3.4 各数据源到TDengine的参考示例 + +下面是一些数据源到TDengine进行数据迁移的示例 + +| 数据迁移示例 | 配置的示例 | +| ------------------ | ------------------------------------------------------------ | +| TDengine到TDengine | [超级表到超级表，指定tbname](../src/test/resources/t2t-1.json) | +| TDengine到TDengine | [超级表到超级表，不指定tbname](../src/test/resources/t2t-2.json) | +| TDengine到TDengine | [超级表到子表](../src/test/resources/t2t-3.json) | +| TDengine到TDengine | [普通表到普通表](../src/test/resources/t2t-4.json) | +| RDBMS到TDengine | [普通表到超级表，指定tbname](../src/test/resources/dm2t-1.json) | +| RDBMS到TDengine | [普通表到超级表，不指定tbname](../src/test/resources/dm2t-2.json) | +| RDBMS到TDengine | [普通表到子表](../src/test/resources/dm2t-3.json) | +| RDBMS到TDengine | [普通表到普通表](../src/test/resources/dm2t-4.json) | +| OpenTSDB到TDengine | [metric到普通表](../src/test/resources/o2t-1.json) | + + + ## 4 性能报告 @@ -362,44 +226,29 @@ CREATE TABLE IF NOT EXISTS weather( 说明： -1. 这里的单表，主键类型为 bigint(20),自增。 -2. batchSize 和 通道个数，对性能影响较大。 -3. 16通道，4096批量提交时，出现 full gc 2次。 +1. #### 4.2.4 性能测试小结 + + ## 5 约束限制 -1. 本插件自动创建超级表时NCHAR类型的长度固定为64，对于包含长度大于64的字符串的数据源，将不支持。 -2. 标签列不能包含null值，如果包含会被过滤掉。 +1. + + ## FAQ -### 如何选取要同步的数据的范围？ - -数据范围的选取在Reader插件端配置，对于不同的Reader插件配置方法往往不同。比如对于mysqlreader， 可以用sql语句指定数据范围。对于opentsdbreader, 用beginDateTime和endDateTime两个配置项指定数据范围。 - -### 如何一次导入多张源表？ - -如果Reader插件支持一次读多张表，Writer插件就能一次导入多张表。如果Reader不支持多多张表，可以建多个job，分别导入。Writer插件只负责写数据。 - ### 一张源表导入之后对应TDengine中多少张表？ 这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 ### 源表和目标表的字段顺序一致吗？ -TDengine要求每个表第一列是时间戳列，后边是普通字段，最后是标签列。如果源表不是这个顺序，插件在自动建表时会自动调整。 +是的，TDengineWriter按照column中字段的顺序解析来自datax的数据。 ### 插件如何确定各列的数据类型？ 根据收到的第一批数据自动推断各列的类型。 - -### 为什么插入10年前的数据会抛异常`TDengine ERROR (2350): failed to execute batch bind` ? - -因为创建数据库的时候，默认保留10年的数据。可以手动指定要保留多长时间的数据，比如:`CREATE DATABASE power KEEP 36500;`。 - -### 如果编译的时候某些插件的依赖找不到怎么办？ - -如果这个插件不是必须的，可以注释掉根目录下的pom.xml中的对应插件。 \ No newline at end of file From 7436176b27a6f3120b05e7b8cf73826ab95e2112 Mon Sep 17 00:00:00 2001 From: zyyang Date: Sun, 20 Feb 2022 16:49:48 +0800 Subject: [PATCH 098/114] change doc --- tdenginewriter/doc/tdenginewriter-CN.md | 10 +- tdenginewriter/doc/tdenginewriter.md | 439 ++++++++---------------- 2 files changed, 141 insertions(+), 308 deletions(-) diff --git a/tdenginewriter/doc/tdenginewriter-CN.md b/tdenginewriter/doc/tdenginewriter-CN.md index 910425c..d06669b 100644 --- a/tdenginewriter/doc/tdenginewriter-CN.md +++ b/tdenginewriter/doc/tdenginewriter-CN.md @@ -140,7 +140,7 @@ create table test.weather (ts timestamp, temperature int, humidity double) tags( * 必选：是 * 默认值：无 * batchSize - * 描述： + * 描述：每batchSize条record为一个batch进行写入 * 必选：否 * 默认值：1 * ignoreTagsUnmatched @@ -241,14 +241,6 @@ datax中的数据类型，可以映射到TDengine的数据类型 ## FAQ -### 一张源表导入之后对应TDengine中多少张表？ - -这是由tagColumn决定的，如果所有tag列的值都相同，那么目标表只有一个。源表有多少不同的tag组合，目标超级表就有多少子表。 - ### 源表和目标表的字段顺序一致吗？ 是的，TDengineWriter按照column中字段的顺序解析来自datax的数据。 - -### 插件如何确定各列的数据类型？ - -根据收到的第一批数据自动推断各列的类型。 diff --git a/tdenginewriter/doc/tdenginewriter.md b/tdenginewriter/doc/tdenginewriter.md index 6f19c11..ba6d1ea 100644 --- a/tdenginewriter/doc/tdenginewriter.md +++ b/tdenginewriter/doc/tdenginewriter.md @@ -4,18 +4,42 @@ ## 1 Quick Introduction -TDengineWriter Plugin writes data to [TDengine](https://www.taosdata.com/en/). It can be used to offline synchronize data from other databases to TDengine. +The TDengineWriter plugin enables writing data to the target table of the TDengine database. At the bottom level, TDengineWriter connects TDengine through JDBC, executes insert statement /schemaless statement according to TDengine SQL syntax, and writes data to TDengine. + +TDengineWriter can be used as a data migration tool for DBAs to import data from other databases into TDengine. + + ## 2 Implementation -TDengineWriter get records from DataX Framework that are generated from reader side. It has two whiting strategies: +TDengineWriter obtains the protocol data generated by Reader through DataX framework, connects to TDengine through JDBC Driver, executes insert statement /schemaless statement, and writes the data to TDengine. + +In TDengine, table can be divided into super table, sub-table and ordinary table. Super table and sub-table include Colum and Tag. The value of tag column of sub-table is fixed value. (details please refer to: [data model](https://www.taosdata.com/docs/cn/v2.0/architecture#model)) + +The TDengineWriter can write data to super tables, sub-tables, and ordinary tables using the following methods based on the type of the table and whether the column parameter contains TBName: + +1. Table is a super table and column specifies tbname: use the automatic insert statement to create the table and use tbname as the name of the sub-table. + +2. Table is a super table and column does not contain tbname: use schemaless to write the table. TDengine will auto-create a tbname based on the super table name and tag value. + +3. Table is a sub-table: Use insert statement to write, ignoreTagUnmatched parameter is true, ignore data in record whose tag value is inconsistent with that of table. + +4. Table is a common table: use insert statement to write data. + -1. For data from OpenTSDBReader which is in json format, to leverage the new feature of TDengine Server that support writing json data directly called [schemaless writing](https://www.taosdata.com/cn/documentation/insert#schemaless), we use JNI to call functions in `taos.lib` or `taos.dll`.(Since the feature was not included in taos-jdbcdrive until version 2.0.36). -2. For other data sources, we use [taos-jdbcdriver](https://www.taosdata.com/cn/documentation/connector/java) to write data. If the target table is not exists beforehand, then it will be created automatically according to your configuration. ## 3 Features Introduction -### 3.1 From OpenTSDB to TDengine -#### 3.1.1 Sample Setting +### 3.1 Sample +Configure a job to write to TDengine + +Create a supertable on TDengine: + +```sql +create database if not exists test; +create table test.weather (ts timestamp, temperature int, humidity double) tags(is_normal bool, device_id binary(100), address nchar(100)); +``` + +Write data to TDengine using the following Job configuration: ```json { @@ -23,286 +47,65 @@ TDengineWriter get records from DataX Framework that are generated from reader s "content": [ { "reader": { - "name": "opentsdbreader", + "name": "streamreader", "parameter": { - "endpoint": "http://192.168.1.180:4242", "column": [ - "weather_temperature" + { + "type": "string", + "value": "tb1" + }, + { + "type": "date", + "value": "2022-02-20 12:00:01" + }, + { + "type": "long", + "random": "0, 10" + }, + { + "type": "double", + "random": "0, 10" + }, + { + "type": "bool", + "random": "0, 50" + }, + { + "type": "bytes", + "value": "abcABC123" + }, + { + "type": "string", + "value": "北京朝阳望京" + } ], - "beginDateTime": "2021-01-01 00:00:00", - "endDateTime": "2021-01-01 01:00:00" + "sliceRecordCount": 1 } }, "writer": { "name": "tdenginewriter", "parameter": { - "host": "192.168.1.180", - "port": 6030, - "dbName": "test", "username": "root", - "password": "taosdata" - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} -``` - -#### 3.1.2 Configuration - -| Parameter | Description | Required | Default | -| --------- | ------------------------------ | -------- | -------- | -| host | host of TDengine | Yes | | -| port | port of TDengine | Yes | | -| username | use name of TDengine | No | root | -| password | password of TDengine | No | taosdata | -| dbName | name of target database | No | | -| batchSize | batch size of insert operation | No | 1 | - - -#### 3.1.3 Type Convert - -| OpenTSDB Type | DataX Type | TDengine Type | -| ---------------- | ---------- | ------------- | -| timestamp | Date | timestamp | -| Integer（value） | Double | double | -| Float（value） | Double | double | -| String（value） | String | binary | -| Integer（tag） | String | binary | -| Float（tag） | String | binary | -| String（tag） | String | binary | - -### 3.2 From MongoDB to TDengine - -#### 3.2.1 Sample Setting -```json -{ - "job": { - "setting": { - "speed": { - "channel": 2 - } - }, - "content": [ - { - "reader": { - "name": "mongodbreader", - "parameter": { - "address": [ - "127.0.0.1:27017" - ], - "userName": "user", - "mechanism": "SCRAM-SHA-1", - "userPassword": "password", - "authDb": "admin", - "dbName": "test", - "collectionName": "stock", - "column": [ - { - "name": "stockID", - "type": "string" - }, - { - "name": "tradeTime", - "type": "date" - }, - { - "name": "lastPrice", - "type": "double" - }, - { - "name": "askPrice1", - "type": "double" - }, - { - "name": "bidPrice1", - "type": "double" - }, - { - "name": "volume", - "type": "int" - } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "localhost", - "port": 6030, - "dbName": "test", - "username": "root", - "password": "taosdata", - "stable": "stock", - "tagColumn": { - "industry": "energy", - "stockID": 0 - }, - "fieldColumn": { - "lastPrice": 2, - "askPrice1": 3, - "bidPrice1": 4, - "volume": 5 - }, - "timestampColumn": { - "tradeTime": 1 - } - } - } - } - ] - } -} -``` - -**Note：the writer part of this setting can also apply to other data source except for OpenTSDB ** - - -#### 3.2.2 Configuration - -| Parameter | Description | Required | Default | Remark | -| --------------- | --------------------------------------------------------------- | ------------------------ | -------- | ------------------- | -| host | host ofTDengine | Yes | | -| port | port of TDengine | Yes | | -| username | username of TDengine | No | root | -| password | password of TDengine | No | taosdata | -| dbName | name of target database | Yes | | -| batchSize | batch size of insert operation | No | 1000 | -| stable | name of target super table | Yes(except for OpenTSDB) | | -| tagColumn | name and position of tag columns in the record from reader, format:{tagName1: tagInd1, tagName2: tagInd2} | No | | index starts with 0 | -| fieldColumn | name and position of data columns in the record from reader, format: {fdName1: fdInd1, fdName2: fdInd2} | No | | | -| timestampColumn | name and position of timestamp column in the record from reader | No | | | - -**Note**: You see that the value of tagColumn "industry" is a fixed string, this ia a good feature of this plugin. Think about this scenario: you have many tables with the structure and one table corresponds to one device. You want to use the device number as a tag in the target super table, then this feature is designed for you. - -#### 3.2.3 Auto table creating -##### 3.2.3.1 Rules - -If all of `tagColumn`, `fieldColumn` and `timestampColumn` are offered in writer configuration, then target super table will be created automatically. -The type of tag columns will always be `NCHAR(64)`. The sample setting above will produce following sql: - -```sql -CREATE STABLE IF NOT EXISTS market_snapshot ( - tadetime TIMESTAMP, - lastprice DOUBLE, - askprice1 DOUBLE, - bidprice1 DOUBLE, - volume INT -) -TAGS( - industry NCHAR(64), - stockID NCHAR(64) -); -``` - -##### 3.2.3.2 Sub-table Creating Rules - -The structure of sub-tables are the same with structure of super table. The names of sub-tables are generated by rules below: -1. combine value of tags like this:`tag_value1!tag_value2!tag_value3`. -2. compute md5 hash hex of above string, named `md5val` -3. use "t_md5val" as sub-table name, in which "t" is fixed prefix. - -#### 3.2.4 Use Pre-created Table - -If you have created super table firstly, then all of tagColumn, fieldColumn and timestampColumn can be omitted. The writer plugin will get table schema by executing `describe stableName`. -The order of columns of records received by this plugin must be the same as the order of columns returned by `describe stableName`. For example, if you have super table as below: -``` - Field | Type | Length | Note | -================================================================================= - ts | TIMESTAMP | 8 | | - current | DOUBLE | 8 | | - location | BINARY | 10 | TAG | -``` -Then the first columns received by this writer plugin must represent timestamp, the second column must represent current with type double, the third column must represent location with internal type string. - -#### 3.2.5 Remarks - -1. Config keys --tagColumn, fieldColumn and timestampColumn, must be presented or omitted at the same time. -2. If above three config keys exist and the target table also exists, then the order of columns defined by the config file and the existed table must be the same. - -#### 3.2.6 Type Convert - -| MongoDB Type | DataX Type | TDengine Type | -| ---------------- | -------------- | ----------------- | -| int, Long | Long | BIGINT | -| double | Double | DOUBLE | -| string, array | String | NCHAR(64) | -| date | Date | TIMESTAMP | -| boolean | Boolean | BOOL | -| bytes | Bytes | BINARY(64) | - -### 3.3 From Relational Database to TDengine - -Take MySQl as example. - -#### 3.3.1 Table Structure in MySQL -```sql -CREATE TABLE IF NOT EXISTS weather( - station varchar(100), - latitude DOUBLE, - longtitude DOUBLE, - `date` DATE, - TMAX int, - TMIN int -) -``` - -#### 3.3.2 Sample Setting - -```json -{ - "job": { - "content": [ - { - "reader": { - "name": "mysqlreader", - "parameter": { - "username": "root", - "password": "passw0rd", + "password": "taosdata", "column": [ - "*" + "tbname", + "ts", + "temperature", + "humidity", + "is_normal", + "device_id", + "address" ], - "splitPk": "station", "connection": [ { "table": [ "weather" ], - "jdbcUrl": [ - "jdbc:mysql://127.0.0.1:3306/test?useSSL=false&useUnicode=true&characterEncoding=utf8" - ] + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test" } - ] - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "host": "127.0.0.1", - "port": 6030, - "dbName": "test", - "username": "root", - "password": "taosdata", - "batchSize": 1000, - "stable": "weather", - "tagColumn": { - "station": 0 - }, - "fieldColumn": { - "latitude": 1, - "longtitude": 2, - "tmax": 4, - "tmin": 5 - }, - "timestampColumn":{ - "date": 3 - } + ], + "batchSize": 100, + "ignoreTagsUnmatched": true } } } @@ -316,41 +119,79 @@ CREATE TABLE IF NOT EXISTS weather( } ``` +### 3.2 Configuration -## 4 Performance Test +* jdbcUrl + * Descrption: Data source JDBC connection information, TDengine JDBC information please refer to: [Java connector](https://www.taosdata.com/docs/cn/v2.0/connector/java#url) + * Required: yes + * Default: none +* username + * Descrption: username + * Required: yes + * Default: none + +* password + * Descrption: password of username + * Required: yes + * Default: none + +* table + * Descrption: A list of table names that should contain all of the columns in the column parameter (except tbname). Note that tbname in column is used as the TDengine sub-table name. + * Required: yes + * Default: none + +* column + * Descrption: A list of field names, the order of the fields should be the column in the record + * Required: yes + * Default: none + +* batchSize + * Descrption: Each batchSize record is written to a batch + * Required: no + * Default: 1 + +* ignoreTagsUnmatched + * Descrption: When table is a sub-table in TDengine, table has a tag value. If the tag value of the data and the tag value of the table are not equal, the data is not written to the table. + * Required: no + * Default: false + + +#### 3.3 Type Convert + +Data types in datax that can be mapped to data types in TDengine + +| DataX Type | TDengine Type | +| ---------- | ----------------------------------------- | +| INT | TINYINT, SMALLINT, INT | +| LONG | TIMESTAMP, TINYINT, SMALLINT, INT, BIGINT | +| DOUBLE | FLOAT, DOUBLE | +| STRING | TIMESTAMP, BINARY, NCHAR | +| BOOL | BOOL | +| DATE | TIMESTAMP | +| BYTES | BINARY | + +### 3.2 From MongoDB to TDengine + +Here are some examples of data sources migrating to TDengine + +| Sample | Configuration | +| -------------------- | ------------------------------------------------------------ | +| TDengine to TDengine | [super table to super table with tbname](../src/test/resources/t2t-1.json) | +| TDengine to TDengine | [super table to super table without tbname](../src/test/resources/t2t-2.json) | +| TDengine to TDengine | [super table to sub-table](../src/test/resources/t2t-3.json) | +| TDengine to TDengine | [table to table](../src/test/resources/t2t-4.json) | +| RDBMS to TDengine | [table to super table with tbname](../src/test/resources/dm2t-1.json) | +| RDBMS to TDengine | [table to super table without tbname](../src/test/resources/dm2t-2.json) | +| RDBMS to TDengine | [table to sub-table](../src/test/resources/dm2t-3.json) | +| RDBMS to TDengine | [table to table](../src/test/resources/dm2t-4.json) | +| OpenTSDB to TDengine | [metric to table](../src/test/resources/o2t-1.json) | + +## 4 Restriction -## 5 Restriction -1. NCHAR type has fixed length 64 when auto creating stable. -2. Rows have null tag values will be dropped. ## FAQ -### How to filter on source table？ - -It depends on reader plugin. For different reader plugins, the way may be different. - -### How to import multiple source tables at once？ - -It depends on reader plugin. If the reader plugin supports reading multiple tables at once, then there is no problem. - -### How many sub-tables will be produced? - -The number of sub-tables is determined by tagColumns, equals to the number of different combinations of tag values. - ### Do columns in source table and columns in target table must be in the same order? -No. TDengine require the first column has timestamp type，which is followed by data columns, followed by tag columns. The writer plugin will create super table in this column order, regardless of origin column orders. - -### How dose the plugin infer the data type of incoming data? - -By the first batch of records it received. - -### Why can't I insert data of 10 years ago? Do this will get error: `TDengine ERROR (2350): failed to execute batch bind`. - -Because the database you created only keep 10 years data by default, you can create table like this: `CREATE DATABASE power KEEP 36500;`, in order to enlarge the time period to 100 years. - - -### What should I do if some dependencies of a plugin can't be found? - -I this plugin is not necessary for you, just remove it from pom.xml under project's root directory. +Yes, TDengineWriter parses the data from the Datax in the order of the fields in the column. From 788cca5ab95d485881c883114f9021887b24229f Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 21 Feb 2022 13:43:57 +0800 Subject: [PATCH 099/114] change --- .../tdenginewriter/Stream2TDengineTest.java | 4 ++-- .../src/test/resources/defaultJob.json | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java index cfd77fd..e54bcbd 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java @@ -12,7 +12,6 @@ import java.sql.Statement; public class Stream2TDengineTest { private String host2 = "192.168.56.105"; - private String precision; @Test public void s2t_case1() throws Throwable { @@ -56,7 +55,8 @@ public class Stream2TDengineTest { stmt.execute("drop database if exists db2"); stmt.execute("create database if not exists db2 precision '" + precision + "'"); stmt.execute("create table db2.stb2(ts1 timestamp, ts2 timestamp,ts3 timestamp,ts4 timestamp,ts5 timestamp," + - "ts6 timestamp,ts7 timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, f6 double," + + "ts6 timestamp,ts7 timestamp, ts8 timestamp, ts9 timestamp, ts10 timestamp, f1 tinyint, f2 smallint," + + "f3 int, f4 bigint, f5 float, f6 double," + "f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp,t2 timestamp,t3 timestamp,t4 timestamp," + "t5 timestamp,t6 timestamp,t7 timestamp, t8 tinyint, t9 smallint, t10 int, t11 bigint, t12 float," + "t13 double, t14 bool, t15 binary(100), t16 nchar(100))"); diff --git a/tdenginewriter/src/test/resources/defaultJob.json b/tdenginewriter/src/test/resources/defaultJob.json index 2548fc9..2a36dfd 100644 --- a/tdenginewriter/src/test/resources/defaultJob.json +++ b/tdenginewriter/src/test/resources/defaultJob.json @@ -41,6 +41,18 @@ "type": "string", "value": "2022-02-20 12:00:07.123456789" }, + { + "type": "long", + "value": 1645329608000 + }, + { + "type": "long", + "value": 1645329609000000 + }, + { + "type": "long", + "value": 1645329610000000000 + }, { "type": "long", "random": "0, 10" @@ -75,7 +87,7 @@ }, { "type": "string", - "random": "0, 10" + "random": "10, 50" }, { "type": "date", @@ -162,6 +174,9 @@ "ts5", "ts6", "ts7", + "ts8", + "ts9", + "ts10", "f1", "f2", "f3", From e1590952ec56402a753a0feb8344eef581efcc93 Mon Sep 17 00:00:00 2001 From: dingbo Date: Mon, 21 Feb 2022 14:37:14 +0800 Subject: [PATCH 100/114] add TDengine to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fe5d122..3cc8f79 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N | | Elasticsearch | | √ |[写](https://github.com/alibaba/DataX/blob/master/elasticsearchwriter/doc/elasticsearchwriter.md)| | 时间序列数据库 | OpenTSDB | √ | |[读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)| | | TSDB | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md)| +| | TDengine | √ | √ |[读](https://github.com/taosdata/DataX/blob/master/tdenginereader/doc/tdenginereader.md) 、[写](https://github.com/taosdata/DataX/blob/master/tdenginewriter/doc/tdenginewriter-CN.md)| # 阿里云DataWorks数据集成 From b55d26dca7630cbcb12fea7e54424268828f3332 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 21 Feb 2022 17:05:56 +0800 Subject: [PATCH 101/114] tdengine writer rebuild --- tdenginewriter/pom.xml | 15 --------------- .../writer/tdenginewriter/DM2TDengineTest.java | 2 ++ .../tdenginewriter/DefaultDataHandlerTest.java | 6 ++---- .../writer/tdenginewriter/Mysql2TDengineTest.java | 2 ++ .../tdenginewriter/Opentsdb2TDengineTest.java | 2 ++ .../writer/tdenginewriter/SchemaManagerTest.java | 7 ++----- .../tdenginewriter/Stream2TDengineTest.java | 3 ++- .../tdenginewriter/TDengine2TDengineTest.java | 2 ++ 8 files changed, 14 insertions(+), 25 deletions(-) diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 97c750d..05fcd4d 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -68,21 +68,6 @@ 0.0.1-SNAPSHOT test - - mysql - mysql-connector-java - 5.1.49 - test - - - - com.dameng - dm-jdbc - 1.8 - system - ${project.basedir}/src/test/resources/DmJdbcDriver18.jar - - diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java index 0eb91de..2ee02ff 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java @@ -2,6 +2,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.core.Engine; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.sql.*; @@ -9,6 +10,7 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.Random; +@Ignore public class DM2TDengineTest { private String host1 = "192.168.0.72"; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java index 3657a4f..75debe3 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java @@ -6,10 +6,7 @@ import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.element.StringColumn; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.transport.record.DefaultRecord; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import java.sql.Connection; import java.sql.DriverManager; @@ -20,6 +17,7 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; +@Ignore public class DefaultDataHandlerTest { private static final String host = "192.168.1.93"; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java index 4a66271..8cbc2d9 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Mysql2TDengineTest.java @@ -2,12 +2,14 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.core.Engine; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.sql.*; import java.text.SimpleDateFormat; import java.util.Random; +@Ignore public class Mysql2TDengineTest { private static final String host1 = "192.168.56.105"; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java index ad326f7..0d212e8 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Opentsdb2TDengineTest.java @@ -2,10 +2,12 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.core.Engine; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import java.sql.*; +@Ignore public class Opentsdb2TDengineTest { @Test diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java index 6df0de1..fac1e0f 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManagerTest.java @@ -1,10 +1,6 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; -import com.alibaba.fastjson.util.TypeUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.*; import java.sql.Connection; import java.sql.DriverManager; @@ -14,6 +10,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +@Ignore public class SchemaManagerTest { private static Connection conn; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java index e54bcbd..613d2aa 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Stream2TDengineTest.java @@ -1,7 +1,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.core.Engine; -import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.sql.Connection; @@ -9,6 +9,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +@Ignore public class Stream2TDengineTest { private String host2 = "192.168.56.105"; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java index 9e95463..18efdc1 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengine2TDengineTest.java @@ -2,12 +2,14 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.core.Engine; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.sql.*; import java.text.SimpleDateFormat; import java.util.Random; +@Ignore public class TDengine2TDengineTest { private static final String host1 = "192.168.56.105"; From ea488d20c4fff89588b5c76a2cfb8048e014e4c1 Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Thu, 24 Feb 2022 16:59:25 +0800 Subject: [PATCH 102/114] Update DefaultDataHandler.java --- .../writer/tdenginewriter/DefaultDataHandler.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 4068be4..970be3a 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -47,6 +47,16 @@ public class DefaultDataHandler implements DataHandler { private Map> columnMetas; + + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + public DefaultDataHandler(Configuration configuration) { this.username = configuration.getString(Key.USERNAME, Constants.DEFAULT_USERNAME); this.password = configuration.getString(Key.PASSWORD, Constants.DEFAULT_PASSWORD); @@ -460,4 +470,4 @@ public class DefaultDataHandler implements DataHandler { "cannot find col: " + colName + " in columns: " + columns); } -} \ No newline at end of file +} From 740bf010e82db0c2a2afa96dde91a8cfdf913619 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 25 Feb 2022 10:11:54 +0800 Subject: [PATCH 103/114] [TS-1219]: incremental synchronization scripts of datax --- ...tdenginereader.md => tdenginereader-CN.md} | 17 +- tdenginereader/pom.xml | 34 +++ .../com/alibaba/datax/plugin/reader/Key.java | 19 -- .../datax/plugin/reader/TDengineReader.java | 216 ++++++++---------- .../reader/TDengineReaderErrorCode.java | 7 +- .../datax/plugin/reader/TDengine2DMTest.java | 87 +++++++ .../plugin/reader/TDengineReaderTest.java | 106 ++++++--- tdenginereader/src/test/resources/t2dm.json | 50 ++++ tdenginewriter/pom.xml | 16 +- .../tdenginewriter/DefaultDataHandler.java | 10 + .../plugin/writer/tdenginewriter/Key.java | 3 + .../writer/tdenginewriter/SchemaManager.java | 2 +- .../writer/tdenginewriter/TDengineWriter.java | 2 +- .../tdenginewriter/DM2TDengineTest.java | 2 +- .../resources/incremental_sync/clean_env.sh | 9 + .../incremental_sync/dm2t-update.json | 63 +++++ .../resources/incremental_sync/dm2t_sync.sh | 57 +++++ .../test/resources/incremental_sync/upload.sh | 5 + 18 files changed, 511 insertions(+), 194 deletions(-) rename tdenginereader/doc/{tdenginereader.md => tdenginereader-CN.md} (88%) delete mode 100644 tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java create mode 100644 tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java create mode 100644 tdenginereader/src/test/resources/t2dm.json create mode 100755 tdenginewriter/src/test/resources/incremental_sync/clean_env.sh create mode 100644 tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json create mode 100755 tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh create mode 100755 tdenginewriter/src/test/resources/incremental_sync/upload.sh diff --git a/tdenginereader/doc/tdenginereader.md b/tdenginereader/doc/tdenginereader-CN.md similarity index 88% rename from tdenginereader/doc/tdenginereader.md rename to tdenginereader/doc/tdenginereader-CN.md index 3c683a6..fe0e487 100644 --- a/tdenginereader/doc/tdenginereader.md +++ b/tdenginereader/doc/tdenginereader-CN.md @@ -37,8 +37,7 @@ TDengineReader 通过TDengine的JDBC driver查询获取数据。 "phase" ], "beginDateTime": "2017-07-14 10:40:00", - "endDateTime": "2017-08-14 10:40:00", - "splitInterval": "1d" + "endDateTime": "2017-08-14 10:40:00" } }, "writer": { @@ -79,17 +78,12 @@ TDengineReader 通过TDengine的JDBC driver查询获取数据。 * 必选：是
* 默认值：无
* **beginDateTime** - * 描述：数据的开始时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss
- * 必选：是
+ * 描述：数据的开始时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss，如果不填为全量同步
+ * 必选：否
* 默认值：无
* **endDateTime** - * 描述：数据的结束时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss
- * 必选：是
- * 默认值：无
-* **splitInterval** - * 描述：按照splitInterval来划分task, 每splitInterval创建一个task。例如，20d代表按照每20天的数据划分为1个task。 - 可以配置的时间单位为：d（天）, h（小时）, m（分钟）, s（秒）
- * 必选：是
+ * 描述：数据的结束时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss，如果不填为全量同步
+ * 必选：否
* 默认值：无
### 3.3 类型转换 @@ -106,7 +100,6 @@ TDengineReader 通过TDengine的JDBC driver查询获取数据。 | BINARY | Bytes | | NCHAR | String | - ## 4 性能报告 ### 4.1 环境准备 diff --git a/tdenginereader/pom.xml b/tdenginereader/pom.xml index ca444bc..69ea0c0 100644 --- a/tdenginereader/pom.xml +++ b/tdenginereader/pom.xml @@ -29,10 +29,29 @@ + + com.alibaba + fastjson + 1.2.78 + + + + com.alibaba.datax.tdenginewriter + tdenginewriter + 0.0.1-SNAPSHOT + compile + + com.taosdata.jdbc taos-jdbcdriver 2.0.37 + + + com.alibaba + fastjson + + @@ -47,6 +66,21 @@ 0.0.1-SNAPSHOT compile + + + com.alibaba.datax + datax-core + 0.0.1-SNAPSHOT + test + + + + com.dameng + dm-jdbc + 1.8 + system + ${project.basedir}/src/test/resources/DmJdbcDriver18.jar + diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java deleted file mode 100644 index 95b5538..0000000 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/Key.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.alibaba.datax.plugin.reader; - -public class Key { - - public static final String JDBC_URL = "jdbcUrl"; -// public static final String HOST = "host"; -// public static final String PORT = "port"; -// public static final String DB = "db"; - public static final String TABLE = "table"; - public static final String USER = "username"; - public static final String PASSWORD = "password"; - public static final String CONNECTION = "connection"; -// public static final String SQL = "sql"; - public static final String BEGIN_DATETIME = "beginDateTime"; - public static final String END_DATETIME = "endDateTime"; - public static final String SPLIT_INTERVAL = "splitInterval"; - public static final String COLUMN = "column"; - public static final String MANDATORY_ENCODING = "mandatoryEncoding"; -} diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index f00d879..6c2f122 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -1,13 +1,11 @@ package com.alibaba.datax.plugin.reader; -import com.alibaba.datax.common.constant.CommonConstant; import com.alibaba.datax.common.element.*; import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordSender; import com.alibaba.datax.common.spi.Reader; import com.alibaba.datax.common.util.Configuration; -import com.alibaba.datax.plugin.rdbms.util.DataBaseType; -import com.alibaba.fastjson.JSON; +import com.alibaba.datax.plugin.writer.tdenginewriter.Key; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,84 +20,77 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; public class TDengineReader extends Reader { + private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static class Job extends Reader.Job { - private static final Logger LOG = LoggerFactory.getLogger(Job.class); private Configuration originalConfig; @Override public void init() { this.originalConfig = super.getPluginJobConf(); - // check user - String user = this.originalConfig.getString(Key.USER); - if (StringUtils.isBlank(user)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.USER + "] is not set."); + // check username + String username = this.originalConfig.getString(Key.USERNAME); + if (StringUtils.isBlank(username)) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.USERNAME + "] is not set."); // check password String password = this.originalConfig.getString(Key.PASSWORD); if (StringUtils.isBlank(password)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.PASSWORD + "] is not set."); - - SimpleDateFormat format = new SimpleDateFormat(DATETIME_FORMAT); - // check beginDateTime - String beginDatetime = this.originalConfig.getString(Key.BEGIN_DATETIME); - if (StringUtils.isBlank(beginDatetime)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] is not set."); - Long start; - try { - start = format.parse(beginDatetime).getTime(); - } catch (ParseException e) { - throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); - } - - // check endDateTime - String endDatetime = this.originalConfig.getString(Key.END_DATETIME); - if (StringUtils.isBlank(endDatetime)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.END_DATETIME + "] is not set."); - Long end; - try { - end = format.parse(endDatetime).getTime(); - } catch (ParseException e) { - throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.END_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); - } - if (start >= end) - throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.BEGIN_DATETIME + "] should be less than the parameter [" + Key.END_DATETIME + "]."); - - // check splitInterval - String splitInterval = this.originalConfig.getString(Key.SPLIT_INTERVAL); - Long split; - if (StringUtils.isBlank(splitInterval)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.SPLIT_INTERVAL + "] is not set."); - try { - split = parseSplitInterval(splitInterval); - } catch (Exception e) { - throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "The parameter [" + Key.SPLIT_INTERVAL + "] should be like: \"123d|h|m|s\", error: " + e.getMessage()); - } - - this.originalConfig.set(Key.BEGIN_DATETIME, start); - this.originalConfig.set(Key.END_DATETIME, end); - this.originalConfig.set(Key.SPLIT_INTERVAL, split); + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.PASSWORD + "] is not set."); // check connection List connection = this.originalConfig.getList(Key.CONNECTION); if (connection == null || connection.isEmpty()) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set."); + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.CONNECTION + "] is not set."); for (int i = 0; i < connection.size(); i++) { Configuration conn = Configuration.from(connection.get(i).toString()); List table = conn.getList(Key.TABLE); if (table == null || table.isEmpty()) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.TABLE + "] of connection[" + (i + 1) + "] is not set."); + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.TABLE + "] of connection[" + (i + 1) + "] is not set."); String jdbcUrl = conn.getString(Key.JDBC_URL); if (StringUtils.isBlank(jdbcUrl)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.JDBC_URL + "] of connection[" + (i + 1) + "] is not set."); + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.JDBC_URL + "] of connection[" + (i + 1) + "] is not set."); } - // check column List column = this.originalConfig.getList(Key.COLUMN); if (column == null || column.isEmpty()) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set or is empty."); + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.CONNECTION + "] is not set or is empty."); + + SimpleDateFormat format = new SimpleDateFormat(DATETIME_FORMAT); + // check beginDateTime + String beginDatetime = this.originalConfig.getString(Key.BEGIN_DATETIME); + long start = Long.MIN_VALUE; + if (!StringUtils.isBlank(beginDatetime)) { + try { + start = format.parse(beginDatetime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.BEGIN_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); + } + } + // check endDateTime + String endDatetime = this.originalConfig.getString(Key.END_DATETIME); + long end = Long.MAX_VALUE; + if (!StringUtils.isBlank(endDatetime)) { + try { + end = format.parse(endDatetime).getTime(); + } catch (ParseException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.END_DATETIME + "] needs to conform to the [" + DATETIME_FORMAT + "] format."); + } + } + if (start >= end) + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, + "The parameter [" + Key.BEGIN_DATETIME + "] should be less than the parameter [" + Key.END_DATETIME + "]."); + } @Override @@ -110,78 +101,58 @@ public class TDengineReader extends Reader { @Override public List split(int adviceNumber) { List configurations = new ArrayList<>(); - // do split - Long start = this.originalConfig.getLong(Key.BEGIN_DATETIME); - Long end = this.originalConfig.getLong(Key.END_DATETIME); - Long split = this.originalConfig.getLong(Key.SPLIT_INTERVAL); - List conns = this.originalConfig.getList(Key.CONNECTION); - - for (Long ts = start; ts < end; ts += split) { - for (int i = 0; i < conns.size(); i++) { - Configuration clone = this.originalConfig.clone(); - clone.remove(Key.SPLIT_INTERVAL); - - clone.set(Key.BEGIN_DATETIME, ts); - clone.set(Key.END_DATETIME, Math.min(ts + split, end)); - - Configuration conf = Configuration.from(conns.get(i).toString()); - String jdbcUrl = conf.getString(Key.JDBC_URL); - clone.set(Key.JDBC_URL, jdbcUrl); - clone.set(Key.TABLE, conf.getList(Key.TABLE)); - - // 抽取 jdbcUrl 中的 ip/port 进行资源使用的打标，以提供给 core 做有意义的 shuffle 操作 - clone.set(CommonConstant.LOAD_BALANCE_RESOURCE_MARK, DataBaseType.parseIpFromJdbcUrl(jdbcUrl)); - clone.remove(Key.CONNECTION); - - configurations.add(clone); - LOG.info("Configuration: {}", JSON.toJSONString(clone)); - } + List connectionList = this.originalConfig.getList(Key.CONNECTION); + for (Object conn : connectionList) { + Configuration clone = this.originalConfig.clone(); + Configuration conf = Configuration.from(conn.toString()); + String jdbcUrl = conf.getString(Key.JDBC_URL); + clone.set(Key.JDBC_URL, jdbcUrl); + clone.set(Key.TABLE, conf.getList(Key.TABLE)); + clone.remove(Key.CONNECTION); + configurations.add(clone); } + + LOG.info("Configuration: {}", configurations); return configurations; } } public static class Task extends Reader.Task { private static final Logger LOG = LoggerFactory.getLogger(Task.class); - private Connection conn; - private Long startTime; - private Long endTime; + private Configuration readerSliceConfig; + private Connection conn; private List tables; private List columns; - private String mandatoryEncoding; + + private String startTime; + private String endTime; @Override public void init() { - Configuration readerSliceConfig = super.getPluginJobConf(); - LOG.info("getPluginJobConf: {}", JSON.toJSONString(readerSliceConfig)); + this.readerSliceConfig = super.getPluginJobConf(); + LOG.info("getPluginJobConf: {}", readerSliceConfig); String url = readerSliceConfig.getString(Key.JDBC_URL); if (StringUtils.isBlank(url)) throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.JDBC_URL + "] is not set."); - - tables = readerSliceConfig.getList(Key.TABLE, String.class); - columns = readerSliceConfig.getList(Key.COLUMN, String.class); - - String user = readerSliceConfig.getString(Key.USER); + String user = readerSliceConfig.getString(Key.USERNAME); String password = readerSliceConfig.getString(Key.PASSWORD); - try { - conn = DriverManager.getConnection(url, user, password); + this.conn = DriverManager.getConnection(url, user, password); } catch (SQLException e) { throw DataXException.asDataXException(TDengineReaderErrorCode.CONNECTION_FAILED, - "The parameter [" + Key.JDBC_URL + "] : " + url + " failed to connect since: " + e.getMessage()); + "The parameter [" + Key.JDBC_URL + "] : " + url + " failed to connect since: " + e.getMessage(), e); } - this.mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, ""); - - startTime = readerSliceConfig.getLong(Key.BEGIN_DATETIME); - endTime = readerSliceConfig.getLong(Key.END_DATETIME); + this.tables = readerSliceConfig.getList(Key.TABLE, String.class); + this.columns = readerSliceConfig.getList(Key.COLUMN, String.class); + this.startTime = readerSliceConfig.getString(Key.BEGIN_DATETIME); + this.endTime = readerSliceConfig.getString(Key.END_DATETIME); } - @Override public void destroy() { @@ -189,18 +160,31 @@ public class TDengineReader extends Reader { @Override public void startRead(RecordSender recordSender) { + try (Statement stmt = conn.createStatement()) { for (String table : tables) { - String sql = "select " + StringUtils.join(columns, ",") + " from " + table - + " where _c0 >= " + startTime + " and _c0 < " + endTime; + StringBuilder sb = new StringBuilder(); + sb.append("select ").append(StringUtils.join(columns, ",")) + .append(" from ").append(table).append(" "); + + if (StringUtils.isBlank(startTime)) { + sb.append("where _c0 >= ").append(Long.MIN_VALUE); + } else { + sb.append("where _c0 >= '").append(startTime).append("'"); + } + if (!StringUtils.isBlank(endTime)) { + sb.append(" and _c0 < '").append(endTime).append("'"); + } + + String sql = sb.toString(); ResultSet rs = stmt.executeQuery(sql); - ResultSetMetaData metaData = rs.getMetaData(); while (rs.next()) { - transportOneRecord(recordSender, rs, metaData, metaData.getColumnCount(), this.mandatoryEncoding); + Record record = buildRecord(recordSender, rs, "UTF-8"); + recordSender.sendToWriter(record); } } } catch (SQLException e) { - throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + throw DataXException.asDataXException(TDengineReaderErrorCode.RUNTIME_EXCEPTION, e.getMessage(), e); } finally { try { if (conn != null) @@ -211,17 +195,11 @@ public class TDengineReader extends Reader { } } - private Record transportOneRecord(RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, int columnCount, String mandatoryEncoding) { - Record record = buildRecord(recordSender, rs, metaData, columnCount, mandatoryEncoding); - recordSender.sendToWriter(record); - return record; - } - - private Record buildRecord(RecordSender recordSender, ResultSet rs, ResultSetMetaData metaData, int columnCount, String mandatoryEncoding) { + private Record buildRecord(RecordSender recordSender, ResultSet rs, String mandatoryEncoding) { Record record = recordSender.createRecord(); - try { - for (int i = 1; i <= columnCount; i++) { + ResultSetMetaData metaData = rs.getMetaData(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { int columnType = metaData.getColumnType(i); switch (columnType) { case Types.SMALLINT: @@ -261,16 +239,16 @@ public class TDengineReader extends Reader { } } - private static final long second = 1000; - private static final long minute = 60 * second; - private static final long hour = 60 * minute; - private static final long day = 24 * hour; - private static Long parseSplitInterval(String splitInterval) throws Exception { + final long second = 1000; + final long minute = 60 * second; + final long hour = 60 * minute; + final long day = 24 * hour; + Pattern compile = Pattern.compile("^(\\d+)([dhms])$"); Matcher matcher = compile.matcher(splitInterval); while (matcher.find()) { - Long value = Long.valueOf(matcher.group(1)); + long value = Long.parseLong(matcher.group(1)); if (value == 0) throw new Exception("invalid splitInterval: 0"); char unit = matcher.group(2).charAt(0); diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java index 68bc11e..b784ab0 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReaderErrorCode.java @@ -4,9 +4,10 @@ import com.alibaba.datax.common.spi.ErrorCode; public enum TDengineReaderErrorCode implements ErrorCode { - REQUIRED_VALUE("TDengineReader-00", "缺失必要的值"), - ILLEGAL_VALUE("TDengineReader-01", "值非法"), - CONNECTION_FAILED("TDengineReader-02", "连接错误"); + REQUIRED_VALUE("TDengineReader-00", "parameter value is missing"), + ILLEGAL_VALUE("TDengineReader-01", "invalid parameter value"), + CONNECTION_FAILED("TDengineReader-02", "connection error"), + RUNTIME_EXCEPTION("TDengineWriter-03", "runtime exception"); private final String code; private final String description; diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java new file mode 100644 index 0000000..d9099e0 --- /dev/null +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java @@ -0,0 +1,87 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.core.Engine; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Random; + +@Ignore +public class TDengine2DMTest { + private static final String host1 = "192.168.56.105"; + private static final String host2 = "192.168.0.72"; + + private final Random random = new Random(System.currentTimeMillis()); + + @Test + public void t2dm_case01() throws Throwable { + // given + createSupTable("ms"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2dm.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void t2dm_case02() throws Throwable { + // given + createSupTable("us"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2dm.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void t2dm_case03() throws Throwable { + // given + createSupTable("ns"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2dm.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + private void createSupTable(String precision) throws SQLException { + final String url = "jdbc:TAOS-RS://" + host1 + ":6041/"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db1"); + stmt.execute("create database if not exists db1 precision '" + precision + "'"); + stmt.execute("create table db1.stb1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, " + + "f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, t3 smallint, " + + "t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(100))"); + + for (int i = 1; i <= 10; i++) { + stmt.execute("insert into db1.tb" + i + " using db1.stb1 tags(now, " + random.nextInt(10) + "," + + random.nextInt(10) + "," + random.nextInt(10) + "," + random.nextInt(10) + "," + + random.nextFloat() + "," + random.nextDouble() + "," + random.nextBoolean() + ",'abcABC123'," + + "'北京朝阳望京') values(now+" + i + "s, " + random.nextInt(10) + "," + random.nextInt(10) + "," + + +random.nextInt(10) + "," + random.nextInt(10) + "," + random.nextFloat() + "," + + random.nextDouble() + "," + random.nextBoolean() + ",'abcABC123','北京朝阳望京')"); + } + stmt.close(); + } + + final String url2 = "jdbc:dm://" + host2 + ":5236"; + try (Connection conn = DriverManager.getConnection(url2, "TESTUSER", "test123456")) { + conn.setAutoCommit(true); + Statement stmt = conn.createStatement(); + stmt.execute("drop table if exists stb2"); + stmt.execute("create table stb2(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, " + + "f6 double, f7 BIT, f8 VARCHAR(100), f9 VARCHAR2(200), t1 timestamp, t2 tinyint, t3 smallint, " + + "t4 int, t5 bigint, t6 float, t7 double, t8 BIT, t9 VARCHAR(100), t10 VARCHAR2(200))"); + } + } + +} diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java index d51d422..da980c5 100644 --- a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java @@ -1,72 +1,118 @@ package com.alibaba.datax.plugin.reader; import com.alibaba.datax.common.util.Configuration; +import com.alibaba.datax.plugin.writer.tdenginewriter.Key; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.List; public class TDengineReaderTest { - TDengineReader.Job job; - - @Before - public void before() { - job = new TDengineReader.Job(); + @Test + public void jobInit_case01() { + // given + TDengineReader.Job job = new TDengineReader.Job(); Configuration configuration = Configuration.from("{" + - "\"user\": \"root\"," + + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + "\"beginDateTime\": \"2021-01-01 00:00:00\"," + - "\"endDateTime\": \"2021-01-01 12:00:00\"," + - "\"splitInterval\": \"1h\"" + + "\"endDateTime\": \"2021-01-01 12:00:00\"" + "}"); job.setPluginJobConf(configuration); - } - @Test - public void jobInit() throws ParseException { // when job.init(); // assert Configuration conf = job.getPluginJobConf(); - Assert.assertEquals("root", conf.getString("user")); + Assert.assertEquals("root", conf.getString(Key.USERNAME)); Assert.assertEquals("taosdata", conf.getString("password")); Assert.assertEquals("weather", conf.getString("connection[0].table[0]")); Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl")); + Assert.assertEquals("2021-01-01 00:00:00", conf.getString("beginDateTime")); + Assert.assertEquals("2021-01-01 12:00:00", conf.getString("endDateTime")); + } - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Long start = sdf.parse("2021-01-01 00:00:00").getTime(); - Assert.assertEquals(start, conf.getLong("beginDateTime")); + @Test + public void jobInit_case02() { + // given + TDengineReader.Job job = new TDengineReader.Job(); + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]" + + "}"); + job.setPluginJobConf(configuration); - Long end = sdf.parse("2021-01-01 12:00:00").getTime(); - Assert.assertEquals(end, conf.getLong("endDateTime")); + // when + job.init(); - Assert.assertEquals(new Long(3600 * 1000), conf.getLong("splitInterval")); + // assert + Configuration conf = job.getPluginJobConf(); + + Assert.assertEquals("root", conf.getString(Key.USERNAME)); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("weather", conf.getString("connection[0].table[0]")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl")); } @Test - public void jobSplit() { + public void jobSplit_case01() { + // given + TDengineReader.Job job = new TDengineReader.Job(); + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + + "\"beginDateTime\": \"2021-01-01 00:00:00\"," + + "\"endDateTime\": \"2021-01-01 12:00:00\"" + + "}"); + job.setPluginJobConf(configuration); + // when job.init(); List configurationList = job.split(1); // assert - Assert.assertEquals(12, configurationList.size()); - for (int i = 0; i < configurationList.size(); i++) { - Configuration conf = configurationList.get(i); - Assert.assertEquals("root", conf.getString("user")); - Assert.assertEquals("taosdata", conf.getString("password")); - Assert.assertEquals("weather", conf.getString("table[0]")); - Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); - } + Assert.assertEquals(1, configurationList.size()); + Configuration conf = configurationList.get(0); + Assert.assertEquals("root", conf.getString("username")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("weather", conf.getString("table[0]")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); + + } + + @Test + public void jobSplit_case02() { + // given + TDengineReader.Job job = new TDengineReader.Job(); + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + + "}"); + job.setPluginJobConf(configuration); + + // when + job.init(); + List configurationList = job.split(1); + + // assert + Assert.assertEquals(1, configurationList.size()); + Configuration conf = configurationList.get(0); + Assert.assertEquals("root", conf.getString("username")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("weather", conf.getString("table[0]")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); } } \ No newline at end of file diff --git a/tdenginereader/src/test/resources/t2dm.json b/tdenginereader/src/test/resources/t2dm.json new file mode 100644 index 0000000..b2cf91e --- /dev/null +++ b/tdenginereader/src/test/resources/t2dm.json @@ -0,0 +1,50 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "*" + ], + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1" + } + ] + } + }, + "writer": { + "name": "rdbmswriter", + "parameter": { + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:dm://192.168.0.72:5236" + } + ], + "username": "TESTUSER", + "password": "test123456", + "table": "stb2", + "column": [ + "*" + ] + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/pom.xml b/tdenginewriter/pom.xml index 97c750d..791a4bd 100644 --- a/tdenginewriter/pom.xml +++ b/tdenginewriter/pom.xml @@ -75,14 +75,14 @@ test - - com.dameng - dm-jdbc - 1.8 - system - ${project.basedir}/src/test/resources/DmJdbcDriver18.jar - - + + + + + + + + diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 4068be4..a819615 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -19,6 +19,15 @@ import java.util.Date; import java.util.stream.Collectors; public class DefaultDataHandler implements DataHandler { + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); private String username; @@ -62,6 +71,7 @@ public class DefaultDataHandler implements DataHandler { int count = 0; int affectedRows = 0; + try (Connection conn = DriverManager.getConnection(jdbcUrl, username, password)) { LOG.info("connection[ jdbcUrl: " + jdbcUrl + ", username: " + username + "] established."); // prepare table_name -> table_meta diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index 1d7ee21..b95ff58 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -9,4 +9,7 @@ public class Key { public static final String JDBC_URL = "jdbcUrl"; public static final String COLUMN = "column"; public static final String IGNORE_TAGS_UNMATCHED = "ignoreTagsUnmatched"; + + public static final String BEGIN_DATETIME = "beginDateTime"; + public static final String END_DATETIME = "endDateTime"; } \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index 63d67a2..c48b794 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -17,7 +17,7 @@ public class SchemaManager { private final Connection conn; private TimestampPrecision precision; - SchemaManager(Connection conn) { + public SchemaManager(Connection conn) { this.conn = conn; } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java index 7cc76a7..eb53802 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -25,7 +25,7 @@ public class TDengineWriter extends Writer { this.originalConfig = super.getPluginJobConf(); this.originalConfig.set(PEER_PLUGIN_NAME, getPeerPluginName()); - // check user + // check username String user = this.originalConfig.getString(Key.USERNAME); if (StringUtils.isBlank(user)) throw DataXException.asDataXException(TDengineWriterErrorCode.REQUIRED_VALUE, "The parameter [" diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java index 0eb91de..15f6b1b 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DM2TDengineTest.java @@ -99,7 +99,7 @@ public class DM2TDengineTest { } @Before - public void before() throws SQLException, ClassNotFoundException { + public void before() throws SQLException { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); long ts = System.currentTimeMillis(); diff --git a/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh b/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh new file mode 100755 index 0000000..f3dca7c --- /dev/null +++ b/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +datax_home_dir=$(dirname $(readlink -f "$0")) + +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'drop table if exists db2.stb2;' 192.168.1.93:6041/rest/sql +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create table if not exists db2.stb2 (`ts` TIMESTAMP,`f2` SMALLINT,`f4` BIGINT,`f5` FLOAT,`f6` DOUBLE,`f7` DOUBLE,`f8` BOOL,`f9` NCHAR(100),`f10` NCHAR(200)) TAGS (`f1` TINYINT,`f3` INT);' 192.168.1.93:6041/rest/sql + +rm -f ${datax_home_dir}/log/* +rm -f ${datax_home_dir}/job/*.csv \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json b/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json new file mode 100644 index 0000000..d9285b2 --- /dev/null +++ b/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json @@ -0,0 +1,63 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "rdbmsreader", + "parameter": { + "username": "TESTUSER", + "password": "test123456", + "connection": [ + { + "querySql": [ + "select concat(concat(concat('t', f1), '_'),f3) as tbname,* from stb1" + ], + "jdbcUrl": [ + "jdbc:dm://192.168.0.72:5236" + ] + } + ], + "where": "1=1", + "fetchSize": 1024 + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "tbname", + "ts", + "f1", + "f2", + "f3", + "f4", + "f5", + "f6", + "f7", + "f8", + "f9", + "f10" + ], + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh b/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh new file mode 100755 index 0000000..426c623 --- /dev/null +++ b/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -e +#set -x + +datax_home_dir=$(dirname $(readlink -f "$0")) +table_name="stb1" +update_key="ts" + +while getopts "hd:t:" arg; do + case $arg in + d) + datax_home_dir=$(echo $OPTARG) + ;; + v) + table_name=$(echo $OPTARG) + ;; + h) + echo "Usage: $(basename $0) -d [datax_home_dir] -t [table_name] -k [update_key]" + echo " -h help" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +if [[ -e ${datax_home_dir}/job/${table_name}.csv ]]; then + MAX_TIME=$(cat ${datax_home_dir}/job/${table_name}.csv) +else + MAX_TIME="null" +fi +current_datetime=$(date +"%Y-%m-%d %H:%M:%S") +current_timestamp=$(date +%s) + +if [ "$MAX_TIME" != "null" ]; then + WHERE="${update_key} >= '$MAX_TIME' and ${update_key} < '$current_datetime'" + sed "s/1=1/$WHERE/g" ${datax_home_dir}/job/dm2t-update.json >${datax_home_dir}/job/dm2t_${current_timestamp}.json + echo "incremental data synchronization, from '${MAX_TIME}' to '${current_datetime}'" + python ${datax_home_dir}/bin/datax.py ${datax_home_dir}/job/dm2t_${current_timestamp}.json 1> /dev/null 2>&1 +else + echo "full data synchronization, to '${current_datetime}'" + python ${datax_home_dir}/bin/datax.py ${datax_home_dir}/job/dm2t-update.json 1> /dev/null 2>&1 +fi + +if [[ $? -ne 0 ]]; then + echo "datax migration job falied" +else + echo ${current_datetime} >$datax_home_dir/job/${table_name}.csv + echo "datax migration job success" +fi + +rm -rf ${datax_home_dir}/job/dm2t_${current_timestamp}.json + +#while true; do ./dm2t_sync.sh; sleep 5s; done \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/upload.sh b/tdenginewriter/src/test/resources/incremental_sync/upload.sh new file mode 100755 index 0000000..c7d11ca --- /dev/null +++ b/tdenginewriter/src/test/resources/incremental_sync/upload.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +scp dm2t-update.json root@192.168.56.105:/root/workspace/tmp/datax/job +scp dm2t_sync.sh root@192.168.56.105:/root/workspace/tmp/datax +scp clean_env.sh root@192.168.56.105:/root/workspace/tmp/datax \ No newline at end of file From 3d35c224247e2ecb0d91a25c403a6aafeb7d45fe Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 25 Feb 2022 10:18:04 +0800 Subject: [PATCH 104/114] change --- tdenginereader/pom.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tdenginereader/pom.xml b/tdenginereader/pom.xml index 69ea0c0..319152f 100644 --- a/tdenginereader/pom.xml +++ b/tdenginereader/pom.xml @@ -74,13 +74,13 @@ test - - com.dameng - dm-jdbc - 1.8 - system - ${project.basedir}/src/test/resources/DmJdbcDriver18.jar - + + + + + + + From 3b7bf92edf1957b28625e6f92ccae58f2e38859c Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 2 Mar 2022 16:08:07 +0800 Subject: [PATCH 105/114] invoke load driver in static block --- .../datax/plugin/reader/TDengineReader.java | 33 +++--------- .../resources/incremental_sync/t2dm-jni.json | 50 +++++++++++++++++++ .../incremental_sync/t2dm-restful.json | 50 +++++++++++++++++++ .../test/resources/incremental_sync/upload.sh | 3 +- 4 files changed, 108 insertions(+), 28 deletions(-) create mode 100644 tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json create mode 100644 tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index 6c2f122..aea75f8 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -16,8 +16,6 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class TDengineReader extends Reader { @@ -239,32 +237,13 @@ public class TDengineReader extends Reader { } } - private static Long parseSplitInterval(String splitInterval) throws Exception { - final long second = 1000; - final long minute = 60 * second; - final long hour = 60 * minute; - final long day = 24 * hour; - - Pattern compile = Pattern.compile("^(\\d+)([dhms])$"); - Matcher matcher = compile.matcher(splitInterval); - while (matcher.find()) { - long value = Long.parseLong(matcher.group(1)); - if (value == 0) - throw new Exception("invalid splitInterval: 0"); - char unit = matcher.group(2).charAt(0); - switch (unit) { - case 'd': - return value * day; - default: - case 'h': - return value * hour; - case 'm': - return value * minute; - case 's': - return value * second; - } + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); } - throw new Exception("invalid splitInterval: " + splitInterval); } } diff --git a/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json b/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json new file mode 100644 index 0000000..341f629 --- /dev/null +++ b/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json @@ -0,0 +1,50 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "*" + ], + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": "jdbc:TAOS://192.168.56.105:6030/db1" + } + ] + } + }, + "writer": { + "name": "rdbmswriter", + "parameter": { + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:dm://192.168.0.72:5236" + } + ], + "username": "TESTUSER", + "password": "test123456", + "table": "stb2", + "column": [ + "*" + ] + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json b/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json new file mode 100644 index 0000000..b2cf91e --- /dev/null +++ b/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json @@ -0,0 +1,50 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "*" + ], + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1" + } + ] + } + }, + "writer": { + "name": "rdbmswriter", + "parameter": { + "connection": [ + { + "table": [ + "stb2" + ], + "jdbcUrl": "jdbc:dm://192.168.0.72:5236" + } + ], + "username": "TESTUSER", + "password": "test123456", + "table": "stb2", + "column": [ + "*" + ] + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/upload.sh b/tdenginewriter/src/test/resources/incremental_sync/upload.sh index c7d11ca..9b2a35e 100755 --- a/tdenginewriter/src/test/resources/incremental_sync/upload.sh +++ b/tdenginewriter/src/test/resources/incremental_sync/upload.sh @@ -2,4 +2,5 @@ scp dm2t-update.json root@192.168.56.105:/root/workspace/tmp/datax/job scp dm2t_sync.sh root@192.168.56.105:/root/workspace/tmp/datax -scp clean_env.sh root@192.168.56.105:/root/workspace/tmp/datax \ No newline at end of file +scp clean_env.sh root@192.168.56.105:/root/workspace/tmp/datax +scp \ No newline at end of file From c5c41de1b18ba635a3bf23f62cf770846076acf0 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 2 Mar 2022 17:21:50 +0800 Subject: [PATCH 106/114] change --- .../datax/plugin/reader/TDengineReader.java | 18 +++--- .../resources/incremental_sync/clean_env.sh | 9 --- .../incremental_sync/dm2t-update.json | 63 ------------------- .../resources/incremental_sync/dm2t_sync.sh | 57 ----------------- .../resources/incremental_sync/t2dm-jni.json | 50 --------------- .../incremental_sync/t2dm-restful.json | 50 --------------- .../test/resources/incremental_sync/upload.sh | 6 -- 7 files changed, 10 insertions(+), 243 deletions(-) delete mode 100755 tdenginewriter/src/test/resources/incremental_sync/clean_env.sh delete mode 100644 tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json delete mode 100755 tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh delete mode 100644 tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json delete mode 100644 tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json delete mode 100755 tdenginewriter/src/test/resources/incremental_sync/upload.sh diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index aea75f8..0cf214c 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -127,6 +127,15 @@ public class TDengineReader extends Reader { private String startTime; private String endTime; + static { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + @Override public void init() { this.readerSliceConfig = super.getPluginJobConf(); @@ -237,13 +246,6 @@ public class TDengineReader extends Reader { } } - static { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } + } diff --git a/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh b/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh deleted file mode 100755 index f3dca7c..0000000 --- a/tdenginewriter/src/test/resources/incremental_sync/clean_env.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -datax_home_dir=$(dirname $(readlink -f "$0")) - -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'drop table if exists db2.stb2;' 192.168.1.93:6041/rest/sql -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create table if not exists db2.stb2 (`ts` TIMESTAMP,`f2` SMALLINT,`f4` BIGINT,`f5` FLOAT,`f6` DOUBLE,`f7` DOUBLE,`f8` BOOL,`f9` NCHAR(100),`f10` NCHAR(200)) TAGS (`f1` TINYINT,`f3` INT);' 192.168.1.93:6041/rest/sql - -rm -f ${datax_home_dir}/log/* -rm -f ${datax_home_dir}/job/*.csv \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json b/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json deleted file mode 100644 index d9285b2..0000000 --- a/tdenginewriter/src/test/resources/incremental_sync/dm2t-update.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "rdbmsreader", - "parameter": { - "username": "TESTUSER", - "password": "test123456", - "connection": [ - { - "querySql": [ - "select concat(concat(concat('t', f1), '_'),f3) as tbname,* from stb1" - ], - "jdbcUrl": [ - "jdbc:dm://192.168.0.72:5236" - ] - } - ], - "where": "1=1", - "fetchSize": 1024 - } - }, - "writer": { - "name": "tdenginewriter", - "parameter": { - "username": "root", - "password": "taosdata", - "column": [ - "tbname", - "ts", - "f1", - "f2", - "f3", - "f4", - "f5", - "f6", - "f7", - "f8", - "f9", - "f10" - ], - "connection": [ - { - "table": [ - "stb2" - ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.1.93:6041/db2" - } - ], - "batchSize": 1000, - "ignoreTagsUnmatched": true - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh b/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh deleted file mode 100755 index 426c623..0000000 --- a/tdenginewriter/src/test/resources/incremental_sync/dm2t_sync.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -e -#set -x - -datax_home_dir=$(dirname $(readlink -f "$0")) -table_name="stb1" -update_key="ts" - -while getopts "hd:t:" arg; do - case $arg in - d) - datax_home_dir=$(echo $OPTARG) - ;; - v) - table_name=$(echo $OPTARG) - ;; - h) - echo "Usage: $(basename $0) -d [datax_home_dir] -t [table_name] -k [update_key]" - echo " -h help" - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; - esac -done - -if [[ -e ${datax_home_dir}/job/${table_name}.csv ]]; then - MAX_TIME=$(cat ${datax_home_dir}/job/${table_name}.csv) -else - MAX_TIME="null" -fi -current_datetime=$(date +"%Y-%m-%d %H:%M:%S") -current_timestamp=$(date +%s) - -if [ "$MAX_TIME" != "null" ]; then - WHERE="${update_key} >= '$MAX_TIME' and ${update_key} < '$current_datetime'" - sed "s/1=1/$WHERE/g" ${datax_home_dir}/job/dm2t-update.json >${datax_home_dir}/job/dm2t_${current_timestamp}.json - echo "incremental data synchronization, from '${MAX_TIME}' to '${current_datetime}'" - python ${datax_home_dir}/bin/datax.py ${datax_home_dir}/job/dm2t_${current_timestamp}.json 1> /dev/null 2>&1 -else - echo "full data synchronization, to '${current_datetime}'" - python ${datax_home_dir}/bin/datax.py ${datax_home_dir}/job/dm2t-update.json 1> /dev/null 2>&1 -fi - -if [[ $? -ne 0 ]]; then - echo "datax migration job falied" -else - echo ${current_datetime} >$datax_home_dir/job/${table_name}.csv - echo "datax migration job success" -fi - -rm -rf ${datax_home_dir}/job/dm2t_${current_timestamp}.json - -#while true; do ./dm2t_sync.sh; sleep 5s; done \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json b/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json deleted file mode 100644 index 341f629..0000000 --- a/tdenginewriter/src/test/resources/incremental_sync/t2dm-jni.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "tdenginereader", - "parameter": { - "username": "root", - "password": "taosdata", - "column": [ - "*" - ], - "connection": [ - { - "table": [ - "stb1" - ], - "jdbcUrl": "jdbc:TAOS://192.168.56.105:6030/db1" - } - ] - } - }, - "writer": { - "name": "rdbmswriter", - "parameter": { - "connection": [ - { - "table": [ - "stb2" - ], - "jdbcUrl": "jdbc:dm://192.168.0.72:5236" - } - ], - "username": "TESTUSER", - "password": "test123456", - "table": "stb2", - "column": [ - "*" - ] - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json b/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json deleted file mode 100644 index b2cf91e..0000000 --- a/tdenginewriter/src/test/resources/incremental_sync/t2dm-restful.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "job": { - "content": [ - { - "reader": { - "name": "tdenginereader", - "parameter": { - "username": "root", - "password": "taosdata", - "column": [ - "*" - ], - "connection": [ - { - "table": [ - "stb1" - ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1" - } - ] - } - }, - "writer": { - "name": "rdbmswriter", - "parameter": { - "connection": [ - { - "table": [ - "stb2" - ], - "jdbcUrl": "jdbc:dm://192.168.0.72:5236" - } - ], - "username": "TESTUSER", - "password": "test123456", - "table": "stb2", - "column": [ - "*" - ] - } - } - } - ], - "setting": { - "speed": { - "channel": 1 - } - } - } -} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/incremental_sync/upload.sh b/tdenginewriter/src/test/resources/incremental_sync/upload.sh deleted file mode 100755 index 9b2a35e..0000000 --- a/tdenginewriter/src/test/resources/incremental_sync/upload.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -scp dm2t-update.json root@192.168.56.105:/root/workspace/tmp/datax/job -scp dm2t_sync.sh root@192.168.56.105:/root/workspace/tmp/datax -scp clean_env.sh root@192.168.56.105:/root/workspace/tmp/datax -scp \ No newline at end of file From 8d29a621be7403a8c24d3f41419dd057fa0e1079 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 7 Mar 2022 16:45:08 +0800 Subject: [PATCH 107/114] [TS-1280]: add parameter where and querySql --- tdenginereader/doc/tdenginereader-CN.md | 83 +++++++++++--- .../datax/plugin/reader/TDengineReader.java | 101 ++++++++++-------- .../main/resources/plugin_job_template.json | 6 +- .../datax/plugin/reader/TDengine2DMTest.java | 1 - .../plugin/reader/TDengine2StreamTest.java | 66 ++++++++++++ .../plugin/reader/TDengineReaderTest.java | 53 +++++++-- tdenginereader/src/test/resources/t2dm.json | 4 +- .../src/test/resources/t2stream-1.json | 47 ++++++++ .../src/test/resources/t2stream-2.json | 37 +++++++ .../plugin/writer/tdenginewriter/Key.java | 3 + 10 files changed, 334 insertions(+), 67 deletions(-) create mode 100644 tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2StreamTest.java create mode 100644 tdenginereader/src/test/resources/t2stream-1.json create mode 100644 tdenginereader/src/test/resources/t2stream-2.json diff --git a/tdenginereader/doc/tdenginereader-CN.md b/tdenginereader/doc/tdenginereader-CN.md index fe0e487..aa3751e 100644 --- a/tdenginereader/doc/tdenginereader-CN.md +++ b/tdenginereader/doc/tdenginereader-CN.md @@ -6,12 +6,14 @@ TDengineReader 插件实现了 TDengine 读取数据的功能。 ## 2 实现原理 -TDengineReader 通过TDengine的JDBC driver查询获取数据。 +TDengineReader 通过 TDengine 的 JDBC driver 查询获取数据。 ## 3 功能说明 ### 3.1 配置样例 +* 配置一个从 TDengine 抽取数据作业: + ```json { "job": { @@ -27,7 +29,9 @@ TDengineReader 通过TDengine的JDBC driver查询获取数据。 "table": [ "meters" ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" + "jdbcUrl": [ + "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" + ] } ], "column": [ @@ -36,6 +40,7 @@ TDengineReader 通过TDengine的JDBC driver查询获取数据。 "voltage", "phase" ], + "where": "ts>=0", "beginDateTime": "2017-07-14 10:40:00", "endDateTime": "2017-08-14 10:40:00" } @@ -58,35 +63,89 @@ TDengineReader 通过TDengine的JDBC driver查询获取数据。 } ``` +* 配置一个自定义 SQL 的数据抽取作业： + +```json +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "user": "root", + "password": "taosdata", + "connection": [ + { + "querySql": [ + "select * from test.meters" + ], + "jdbcUrl": [ + "jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP" + ] + } + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} +``` + ### 3.2 参数说明 * **username** - * 描述：TDengine实例的用户名
+ * 描述：TDengine 实例的用户名
* 必选：是
* 默认值：无
* **password** - * 描述：TDengine实例的密码
+ * 描述：TDengine 实例的密码
* 必选：是
* 默认值：无
-* **table** - * 描述：所选取的需要同步的表。使用JSON的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一schema结构， - TDengineReader不予检查表是否同一逻辑表。注意，table必须包含在connection配置单元中。
- * 必选：是
- * 默认值：无
* **jdbcUrl** - * 描述：TDengine数据库的JDBC连接信息。注意，jdbcUrl必须包含在connection配置单元中。JdbcUrl具体请参看TDengine官方文档。 + * 描述：TDengine 数据库的JDBC连接信息。注意，jdbcUrl必须包含在connection配置单元中。JdbcUrl具体请参看TDengine官方文档。 * 必选：是
* 默认值：无
+* **querySql** + * 描述：在有些业务场景下，where 这一配置项不足以描述所筛选的条件，用户可以通过该配置型来自定义筛选SQL。当用户配置了 querySql 后， TDengineReader 就会忽略 table, column, + where, beginDateTime, endDateTime这些配置型，直接使用这个配置项的内容对数据进行筛选。例如需要 进行多表join后同步数据，使用 select a,b from table_a join + table_b on table_a.id = table_b.id
+ * 必选：否
+ * 默认值：无
+* **table** + * 描述：所选取的需要同步的表。使用 JSON 的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一 schema 结构， TDengineReader不予检查表是否同一逻辑表。注意，table必须包含在 + connection 配置单元中。
+ * 必选：是
+ * 默认值：无
+* **where** + * 描述：筛选条件中的 where 子句，TDengineReader 根据指定的column, table, where, begingDateTime, endDateTime 条件拼接 SQL，并根据这个 SQL + 进行数据抽取。
+ * 必选：否
+ * 默认值：无
* **beginDateTime** - * 描述：数据的开始时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss，如果不填为全量同步
+ * 描述：数据的开始时间，Job 迁移从 begineDateTime 到 endDateTime 的数据，格式为 yyyy-MM-dd HH:mm:ss
* 必选：否
* 默认值：无
* **endDateTime** - * 描述：数据的结束时间，Job迁移从begineDateTime到endDateTime的数据，格式为yyyy-MM-dd HH:mm:ss，如果不填为全量同步
+ * 描述：数据的结束时间，Job 迁移从 begineDateTime 到 endDateTime 的数据，格式为 yyyy-MM-dd HH:mm:ss
* 必选：否
* 默认值：无
### 3.3 类型转换 + | TDengine 数据类型 | DataX 内部类型 | | --------------- | ------------- | | TINYINT | Long | diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index 0cf214c..35c82be 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -15,6 +15,8 @@ import java.sql.*; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class TDengineReader extends Reader { @@ -41,26 +43,30 @@ public class TDengineReader extends Reader { "The parameter [" + Key.PASSWORD + "] is not set."); // check connection - List connection = this.originalConfig.getList(Key.CONNECTION); - if (connection == null || connection.isEmpty()) + List connectionList = this.originalConfig.getListConfiguration(Key.CONNECTION); + if (connectionList == null || connectionList.isEmpty()) throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.CONNECTION + "] is not set."); - for (int i = 0; i < connection.size(); i++) { - Configuration conn = Configuration.from(connection.get(i).toString()); - List table = conn.getList(Key.TABLE); - if (table == null || table.isEmpty()) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, - "The parameter [" + Key.TABLE + "] of connection[" + (i + 1) + "] is not set."); - String jdbcUrl = conn.getString(Key.JDBC_URL); - if (StringUtils.isBlank(jdbcUrl)) + for (int i = 0; i < connectionList.size(); i++) { + Configuration conn = connectionList.get(i); + // check jdbcUrl + List jdbcUrlList = conn.getList(Key.JDBC_URL); + if (jdbcUrlList == null || jdbcUrlList.isEmpty()) { throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, "The parameter [" + Key.JDBC_URL + "] of connection[" + (i + 1) + "] is not set."); + } + // check table/querySql + List querySqlList = conn.getList(Key.QUERY_SQL); + if (querySqlList == null || querySqlList.isEmpty()) { + String querySql = conn.getString(Key.QUERY_SQL); + if (StringUtils.isBlank(querySql)) { + List table = conn.getList(Key.TABLE); + if (table == null || table.isEmpty()) + throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, + "The parameter [" + Key.TABLE + "] of connection[" + (i + 1) + "] is not set."); + } + } } - // check column - List column = this.originalConfig.getList(Key.COLUMN); - if (column == null || column.isEmpty()) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, - "The parameter [" + Key.CONNECTION + "] is not set or is empty."); SimpleDateFormat format = new SimpleDateFormat(DATETIME_FORMAT); // check beginDateTime @@ -100,15 +106,17 @@ public class TDengineReader extends Reader { public List split(int adviceNumber) { List configurations = new ArrayList<>(); - List connectionList = this.originalConfig.getList(Key.CONNECTION); - for (Object conn : connectionList) { - Configuration clone = this.originalConfig.clone(); - Configuration conf = Configuration.from(conn.toString()); - String jdbcUrl = conf.getString(Key.JDBC_URL); - clone.set(Key.JDBC_URL, jdbcUrl); - clone.set(Key.TABLE, conf.getList(Key.TABLE)); - clone.remove(Key.CONNECTION); - configurations.add(clone); + List connectionList = this.originalConfig.getListConfiguration(Key.CONNECTION); + for (Configuration conn : connectionList) { + List jdbcUrlList = conn.getList(Key.JDBC_URL, String.class); + for (String jdbcUrl : jdbcUrlList) { + Configuration clone = this.originalConfig.clone(); + clone.set(Key.JDBC_URL, jdbcUrl); + clone.set(Key.TABLE, conn.getList(Key.TABLE)); + clone.set(Key.QUERY_SQL, conn.getList(Key.QUERY_SQL)); + clone.remove(Key.CONNECTION); + configurations.add(clone); + } } LOG.info("Configuration: {}", configurations); @@ -120,12 +128,15 @@ public class TDengineReader extends Reader { private static final Logger LOG = LoggerFactory.getLogger(Task.class); private Configuration readerSliceConfig; + private String mandatoryEncoding; private Connection conn; + private List tables; private List columns; - private String startTime; private String endTime; + private String where; + private List querySql; static { try { @@ -141,12 +152,10 @@ public class TDengineReader extends Reader { this.readerSliceConfig = super.getPluginJobConf(); LOG.info("getPluginJobConf: {}", readerSliceConfig); - String url = readerSliceConfig.getString(Key.JDBC_URL); - if (StringUtils.isBlank(url)) - throw DataXException.asDataXException(TDengineReaderErrorCode.REQUIRED_VALUE, - "The parameter [" + Key.JDBC_URL + "] is not set."); String user = readerSliceConfig.getString(Key.USERNAME); String password = readerSliceConfig.getString(Key.PASSWORD); + + String url = readerSliceConfig.getString(Key.JDBC_URL); try { this.conn = DriverManager.getConnection(url, user, password); } catch (SQLException e) { @@ -158,6 +167,9 @@ public class TDengineReader extends Reader { this.columns = readerSliceConfig.getList(Key.COLUMN, String.class); this.startTime = readerSliceConfig.getString(Key.BEGIN_DATETIME); this.endTime = readerSliceConfig.getString(Key.END_DATETIME); + this.where = readerSliceConfig.getString(Key.WHERE, "_c0 > " + Long.MIN_VALUE); + this.querySql = readerSliceConfig.getList(Key.QUERY_SQL, String.class); + this.mandatoryEncoding = readerSliceConfig.getString(Key.MANDATORY_ENCODING, "UTF-8"); } @Override @@ -167,26 +179,30 @@ public class TDengineReader extends Reader { @Override public void startRead(RecordSender recordSender) { + List sqlList = new ArrayList<>(); - try (Statement stmt = conn.createStatement()) { + if (querySql == null || querySql.isEmpty()) { for (String table : tables) { StringBuilder sb = new StringBuilder(); - sb.append("select ").append(StringUtils.join(columns, ",")) - .append(" from ").append(table).append(" "); - - if (StringUtils.isBlank(startTime)) { - sb.append("where _c0 >= ").append(Long.MIN_VALUE); - } else { - sb.append("where _c0 >= '").append(startTime).append("'"); + sb.append("select ").append(StringUtils.join(columns, ",")).append(" from ").append(table).append(" "); + sb.append("where ").append(where); + if (!StringUtils.isBlank(startTime)) { + sb.append(" and _c0 >= '").append(startTime).append("'"); } if (!StringUtils.isBlank(endTime)) { sb.append(" and _c0 < '").append(endTime).append("'"); } + String sql = sb.toString().trim(); + } + } else { + sqlList.addAll(querySql); + } - String sql = sb.toString(); + try (Statement stmt = conn.createStatement()) { + for (String sql : sqlList) { ResultSet rs = stmt.executeQuery(sql); while (rs.next()) { - Record record = buildRecord(recordSender, rs, "UTF-8"); + Record record = buildRecord(recordSender, rs, mandatoryEncoding); recordSender.sendToWriter(record); } } @@ -239,13 +255,14 @@ public class TDengineReader extends Reader { break; } } - } catch (SQLException | UnsupportedEncodingException e) { - throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "获取或发送数据点的过程中出错！", e); + } catch (SQLException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "database query error！", e); + } catch (UnsupportedEncodingException e) { + throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, "illegal mandatoryEncoding", e); } return record; } } - } diff --git a/tdenginereader/src/main/resources/plugin_job_template.json b/tdenginereader/src/main/resources/plugin_job_template.json index 2c7f4cb..934fe96 100644 --- a/tdenginereader/src/main/resources/plugin_job_template.json +++ b/tdenginereader/src/main/resources/plugin_job_template.json @@ -8,7 +8,9 @@ "table": [ "" ], - "jdbcUrl": "" + "jdbcUrl": [ + "" + ] } ], "column": [ @@ -16,6 +18,6 @@ ], "beginDateTime": "", "endDateTime": "", - "splitInterval": "" + "where": "" } } \ No newline at end of file diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java index d9099e0..e106471 100644 --- a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2DMTest.java @@ -1,7 +1,6 @@ package com.alibaba.datax.plugin.reader; import com.alibaba.datax.core.Engine; -import org.junit.Before; import org.junit.Ignore; import org.junit.Test; diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2StreamTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2StreamTest.java new file mode 100644 index 0000000..f628a64 --- /dev/null +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengine2StreamTest.java @@ -0,0 +1,66 @@ +package com.alibaba.datax.plugin.reader; + +import com.alibaba.datax.core.Engine; +import org.junit.Ignore; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Random; + +@Ignore +public class TDengine2StreamTest { + + private static final String host = "192.168.56.105"; + private static final Random random = new Random(System.currentTimeMillis()); + + @Test + public void case01() throws Throwable { + // given + prepare("ms"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2stream-1.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + @Test + public void case02() throws Throwable { + // given + prepare("ms"); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/t2stream-2.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + + private void prepare(String precision) throws SQLException { + final String url = "jdbc:TAOS-RS://" + host + ":6041/"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists db1"); + stmt.execute("create database if not exists db1 precision '" + precision + "'"); + stmt.execute("create table db1.stb1(ts timestamp, f1 tinyint, f2 smallint, f3 int, f4 bigint, f5 float, " + + "f6 double, f7 bool, f8 binary(100), f9 nchar(100)) tags(t1 timestamp, t2 tinyint, t3 smallint, " + + "t4 int, t5 bigint, t6 float, t7 double, t8 bool, t9 binary(100), t10 nchar(100))"); + + for (int i = 1; i <= 10; i++) { + stmt.execute("insert into db1.tb" + i + " using db1.stb1 tags(now, " + random.nextInt(10) + "," + + random.nextInt(10) + "," + random.nextInt(10) + "," + random.nextInt(10) + "," + + random.nextFloat() + "," + random.nextDouble() + "," + random.nextBoolean() + ",'abcABC123'," + + "'北京朝阳望京') values(now+" + i + "s, " + random.nextInt(10) + "," + random.nextInt(10) + "," + + +random.nextInt(10) + "," + random.nextInt(10) + "," + random.nextFloat() + "," + + random.nextDouble() + "," + random.nextBoolean() + ",'abcABC123','北京朝阳望京')"); + } + stmt.close(); + } + } + + +} diff --git a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java index da980c5..491ddba 100644 --- a/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java +++ b/tdenginereader/src/test/java/com/alibaba/datax/plugin/reader/TDengineReaderTest.java @@ -16,8 +16,9 @@ public class TDengineReaderTest { Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + - "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":[\"jdbc:TAOS-RS://master:6041/test\"]}]," + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + + "\"where\":\"_c0 > 0\"," + "\"beginDateTime\": \"2021-01-01 00:00:00\"," + "\"endDateTime\": \"2021-01-01 12:00:00\"" + "}"); @@ -32,9 +33,10 @@ public class TDengineReaderTest { Assert.assertEquals("root", conf.getString(Key.USERNAME)); Assert.assertEquals("taosdata", conf.getString("password")); Assert.assertEquals("weather", conf.getString("connection[0].table[0]")); - Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl[0]")); Assert.assertEquals("2021-01-01 00:00:00", conf.getString("beginDateTime")); Assert.assertEquals("2021-01-01 12:00:00", conf.getString("endDateTime")); + Assert.assertEquals("_c0 > 0", conf.getString("where")); } @@ -45,8 +47,7 @@ public class TDengineReaderTest { Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + - "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + - "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]" + + "\"connection\": [{\"querySql\":[\"select * from weather\"],\"jdbcUrl\":[\"jdbc:TAOS-RS://master:6041/test\"]}]," + "}"); job.setPluginJobConf(configuration); @@ -58,8 +59,8 @@ public class TDengineReaderTest { Assert.assertEquals("root", conf.getString(Key.USERNAME)); Assert.assertEquals("taosdata", conf.getString("password")); - Assert.assertEquals("weather", conf.getString("connection[0].table[0]")); - Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("connection[0].jdbcUrl[0]")); + Assert.assertEquals("select * from weather", conf.getString("connection[0].querySql[0]")); } @Test @@ -69,8 +70,9 @@ public class TDengineReaderTest { Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + - "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":[\"jdbc:TAOS-RS://master:6041/test\"]}]," + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + + "\"where\":\"_c0 > 0\"," + "\"beginDateTime\": \"2021-01-01 00:00:00\"," + "\"endDateTime\": \"2021-01-01 12:00:00\"" + "}"); @@ -85,6 +87,7 @@ public class TDengineReaderTest { Configuration conf = configurationList.get(0); Assert.assertEquals("root", conf.getString("username")); Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("_c0 > 0", conf.getString("where")); Assert.assertEquals("weather", conf.getString("table[0]")); Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); @@ -97,7 +100,7 @@ public class TDengineReaderTest { Configuration configuration = Configuration.from("{" + "\"username\": \"root\"," + "\"password\": \"taosdata\"," + - "\"connection\": [{\"table\":[\"weather\"],\"jdbcUrl\":\"jdbc:TAOS-RS://master:6041/test\"}]," + + "\"connection\": [{\"querySql\":[\"select * from weather\"],\"jdbcUrl\":[\"jdbc:TAOS-RS://master:6041/test\"]}]," + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + "}"); job.setPluginJobConf(configuration); @@ -111,8 +114,40 @@ public class TDengineReaderTest { Configuration conf = configurationList.get(0); Assert.assertEquals("root", conf.getString("username")); Assert.assertEquals("taosdata", conf.getString("password")); - Assert.assertEquals("weather", conf.getString("table[0]")); + Assert.assertEquals("select * from weather", conf.getString("querySql[0]")); Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); } + @Test + public void jobSplit_case03() { + // given + TDengineReader.Job job = new TDengineReader.Job(); + Configuration configuration = Configuration.from("{" + + "\"username\": \"root\"," + + "\"password\": \"taosdata\"," + + "\"connection\": [{\"querySql\":[\"select * from weather\",\"select * from test.meters\"],\"jdbcUrl\":[\"jdbc:TAOS-RS://master:6041/test\", \"jdbc:TAOS://master:6030/test\"]}]," + + "\"column\": [\"ts\",\"current\",\"voltage\",\"phase\"]," + + "}"); + job.setPluginJobConf(configuration); + + // when + job.init(); + List configurationList = job.split(1); + + // assert + Assert.assertEquals(2, configurationList.size()); + Configuration conf = configurationList.get(0); + Assert.assertEquals("root", conf.getString("username")); + Assert.assertEquals("taosdata", conf.getString("password")); + Assert.assertEquals("select * from weather", conf.getString("querySql[0]")); + Assert.assertEquals("jdbc:TAOS-RS://master:6041/test", conf.getString("jdbcUrl")); + + Configuration conf1 = configurationList.get(1); + Assert.assertEquals("root", conf1.getString("username")); + Assert.assertEquals("taosdata", conf1.getString("password")); + Assert.assertEquals("select * from weather", conf1.getString("querySql[0]")); + Assert.assertEquals("select * from test.meters", conf1.getString("querySql[1]")); + Assert.assertEquals("jdbc:TAOS://master:6030/test", conf1.getString("jdbcUrl")); + } + } \ No newline at end of file diff --git a/tdenginereader/src/test/resources/t2dm.json b/tdenginereader/src/test/resources/t2dm.json index b2cf91e..d87ade0 100644 --- a/tdenginereader/src/test/resources/t2dm.json +++ b/tdenginereader/src/test/resources/t2dm.json @@ -15,7 +15,9 @@ "table": [ "stb1" ], - "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/db1" + "jdbcUrl": [ + "jdbc:TAOS-RS://192.168.56.105:6041/db1" + ] } ] } diff --git a/tdenginereader/src/test/resources/t2stream-1.json b/tdenginereader/src/test/resources/t2stream-1.json new file mode 100644 index 0000000..183ab7e --- /dev/null +++ b/tdenginereader/src/test/resources/t2stream-1.json @@ -0,0 +1,47 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "ts", + "f1", + "f2", + "t1", + "t2" + ], + "connection": [ + { + "table": [ + "stb1" + ], + "jdbcUrl": [ + "jdbc:TAOS-RS://192.168.56.105:6041/db1" + ] + } + ], + "where": "t10 = '北京朝阳望京'", + "beginDateTime": "2022-03-07 12:00:00", + "endDateTime": "2022-03-07 19:00:00" + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginereader/src/test/resources/t2stream-2.json b/tdenginereader/src/test/resources/t2stream-2.json new file mode 100644 index 0000000..15bfe9b --- /dev/null +++ b/tdenginereader/src/test/resources/t2stream-2.json @@ -0,0 +1,37 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "tdenginereader", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "querySql": [ + "select * from stb1 where t10 = '北京朝阳望京' and _c0 >= '2022-03-07 12:00:00' and _c0 < '2022-03-07 19:00:00'" + ], + "jdbcUrl": [ + "jdbc:TAOS-RS://192.168.56.105:6041/db1" + ] + } + ] + } + }, + "writer": { + "name": "streamwriter", + "parameter": { + "encoding": "UTF-8", + "print": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java index b95ff58..1a9358d 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Key.java @@ -12,4 +12,7 @@ public class Key { public static final String BEGIN_DATETIME = "beginDateTime"; public static final String END_DATETIME = "endDateTime"; + public static final String WHERE = "where"; + public static final String QUERY_SQL = "querySql"; + public static final String MANDATORY_ENCODING = "mandatoryEncoding"; } \ No newline at end of file From 2ee8878da1d5075cef77dbd63e6fc4305d2e97e1 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 8 Mar 2022 14:57:03 +0800 Subject: [PATCH 108/114] [TS-1281]: collect dirty data in tdenginewriter --- .../tdenginewriter/DefaultDataHandler.java | 54 +++++++++---- .../writer/tdenginewriter/TDengineWriter.java | 5 +- .../tdenginewriter/Csv2TDengineTest.java | 41 ++++++++++ .../DefaultDataHandlerTest.java | 21 +++-- tdenginewriter/src/test/resources/csv2t.json | 80 +++++++++++++++++++ tdenginewriter/src/test/resources/weather.csv | 10 +++ 6 files changed, 186 insertions(+), 25 deletions(-) create mode 100644 tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Csv2TDengineTest.java create mode 100644 tdenginewriter/src/test/resources/csv2t.json create mode 100644 tdenginewriter/src/test/resources/weather.csv diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index a819615..7854afe 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -30,6 +30,7 @@ public class DefaultDataHandler implements DataHandler { private static final Logger LOG = LoggerFactory.getLogger(DefaultDataHandler.class); + private final TaskPluginCollector taskPluginCollector; private String username; private String password; private String jdbcUrl; @@ -56,7 +57,7 @@ public class DefaultDataHandler implements DataHandler { private Map> columnMetas; - public DefaultDataHandler(Configuration configuration) { + public DefaultDataHandler(Configuration configuration, TaskPluginCollector taskPluginCollector) { this.username = configuration.getString(Key.USERNAME, Constants.DEFAULT_USERNAME); this.password = configuration.getString(Key.PASSWORD, Constants.DEFAULT_PASSWORD); this.jdbcUrl = configuration.getString(Key.JDBC_URL); @@ -64,6 +65,7 @@ public class DefaultDataHandler implements DataHandler { this.tables = configuration.getList(Key.TABLE, String.class); this.columns = configuration.getList(Key.COLUMN, String.class); this.ignoreTagsUnmatched = configuration.getBool(Key.IGNORE_TAGS_UNMATCHED, Constants.DEFAULT_IGNORE_TAGS_UNMATCHED); + this.taskPluginCollector = taskPluginCollector; } @Override @@ -86,14 +88,24 @@ public class DefaultDataHandler implements DataHandler { if (i % batchSize != 0) { recordBatch.add(record); } else { - affectedRows = writeBatch(conn, recordBatch); + try { + affectedRows = writeBatch(conn, recordBatch); + } catch (SQLException e) { + LOG.warn("use one row insert. because:" + e.getMessage()); + affectedRows = writeEachRow(conn, recordBatch); + } recordBatch.clear(); } count++; } if (!recordBatch.isEmpty()) { - affectedRows = writeBatch(conn, recordBatch); + try { + affectedRows = writeBatch(conn, recordBatch); + } catch (SQLException e) { + LOG.warn("use one row insert. because:" + e.getMessage()); + affectedRows = writeEachRow(conn, recordBatch); + } recordBatch.clear(); } } catch (SQLException e) { @@ -107,6 +119,21 @@ public class DefaultDataHandler implements DataHandler { return affectedRows; } + private int writeEachRow(Connection conn, List recordBatch) { + int affectedRows = 0; + for (Record record : recordBatch) { + List recordList = new ArrayList<>(); + recordList.add(record); + try { + affectedRows += writeBatch(conn, recordList); + } catch (SQLException e) { + LOG.error(e.getMessage()); + this.taskPluginCollector.collectDirtyRecord(record, e); + } + } + return affectedRows; + } + /** * table: [ "stb1", "stb2", "tb1", "tb2", "t1" ] * stb1[ts,f1,f2] tags:[t1] @@ -118,7 +145,7 @@ public class DefaultDataHandler implements DataHandler { * 3. 对于tb，拼sql，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, t1] => insert into tb(ts, f1, f2) values(ts, f1, f2) * 4. 对于t，拼sql，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, f3, t1, t2] insert into t(ts, f1, f2, f3, t1, t2) values(ts, f1, f2, f3, t1, t2) */ - public int writeBatch(Connection conn, List recordBatch) { + public int writeBatch(Connection conn, List recordBatch) throws SQLException { int affectedRows = 0; for (String table : tables) { TableMeta tableMeta = tableMetas.get(table); @@ -146,7 +173,7 @@ public class DefaultDataHandler implements DataHandler { * record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) * record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) */ - private int writeBatchToSupTableBySQL(Connection conn, String table, List recordBatch) { + private int writeBatchToSupTableBySQL(Connection conn, String table, List recordBatch) throws SQLException { List columnMetas = this.columnMetas.get(table); StringBuilder sb = new StringBuilder("insert into"); @@ -177,13 +204,11 @@ public class DefaultDataHandler implements DataHandler { return executeUpdate(conn, sql); } - private int executeUpdate(Connection conn, String sql) throws DataXException { + private int executeUpdate(Connection conn, String sql) throws SQLException { int count; try (Statement stmt = conn.createStatement()) { LOG.debug(">>> " + sql); count = stmt.executeUpdate(sql); - } catch (SQLException e) { - throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); } return count; } @@ -227,7 +252,7 @@ public class DefaultDataHandler implements DataHandler { * table: ["stb1"], column: ["ts", "f1", "f2", "t1"] * data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, t1] => schemaless: stb1,t1=t1 f1=f1,f2=f2 ts */ - private int writeBatchToSupTableBySchemaless(Connection conn, String table, List recordBatch) { + private int writeBatchToSupTableBySchemaless(Connection conn, String table, List recordBatch) throws SQLException { int count = 0; TimestampPrecision timestampPrecision = schemaManager.loadDatabasePrecision(); @@ -296,11 +321,8 @@ public class DefaultDataHandler implements DataHandler { default: timestampType = SchemalessTimestampType.NOT_CONFIGURED; } - try { - writer.write(lines, SchemalessProtocolType.LINE, timestampType); - } catch (SQLException e) { - throw DataXException.asDataXException(TDengineWriterErrorCode.RUNTIME_EXCEPTION, e.getMessage()); - } + + writer.write(lines, SchemalessProtocolType.LINE, timestampType); LOG.warn("schemalessWriter does not return affected rows!"); return count; @@ -370,7 +392,7 @@ public class DefaultDataHandler implements DataHandler { * else * insert into tb1 (ts, f1, f2) values( record[idx(ts)], record[idx(f1)], record[idx(f2)]) */ - private int writeBatchToSubTable(Connection conn, String table, List recordBatch) { + private int writeBatchToSubTable(Connection conn, String table, List recordBatch) throws SQLException { List columnMetas = this.columnMetas.get(table); StringBuilder sb = new StringBuilder(); @@ -440,7 +462,7 @@ public class DefaultDataHandler implements DataHandler { * table: ["weather"], column: ["ts, f1, f2, f3, t1, t2"] * sql: insert into weather (ts, f1, f2, f3, t1, t2) values( record[idx(ts), record[idx(f1)], ...) */ - private int writeBatchToNormalTable(Connection conn, String table, List recordBatch) { + private int writeBatchToNormalTable(Connection conn, String table, List recordBatch) throws SQLException { List columnMetas = this.columnMetas.get(table); StringBuilder sb = new StringBuilder(); diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java index eb53802..7398274 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriter.java @@ -2,6 +2,7 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; import com.alibaba.datax.common.exception.DataXException; import com.alibaba.datax.common.plugin.RecordReceiver; +import com.alibaba.datax.common.plugin.TaskPluginCollector; import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; import org.apache.commons.lang3.StringUtils; @@ -81,10 +82,12 @@ public class TDengineWriter extends Writer { private static final Logger LOG = LoggerFactory.getLogger(Task.class); private Configuration writerSliceConfig; + private TaskPluginCollector taskPluginCollector; @Override public void init() { this.writerSliceConfig = getPluginJobConf(); + this.taskPluginCollector = super.getTaskPluginCollector(); } @Override @@ -101,7 +104,7 @@ public class TDengineWriter extends Writer { if (peerPluginName.equals("opentsdbreader")) handler = new OpentsdbDataHandler(this.writerSliceConfig); else - handler = new DefaultDataHandler(this.writerSliceConfig); + handler = new DefaultDataHandler(this.writerSliceConfig, this.taskPluginCollector); long records = handler.handle(lineReceiver, getTaskPluginCollector()); LOG.debug("handle data finished, records: " + records); diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Csv2TDengineTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Csv2TDengineTest.java new file mode 100644 index 0000000..7352c3c --- /dev/null +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/Csv2TDengineTest.java @@ -0,0 +1,41 @@ +package com.alibaba.datax.plugin.writer.tdenginewriter; + +import com.alibaba.datax.core.Engine; +import org.junit.Ignore; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +@Ignore +public class Csv2TDengineTest { + + private static final String host = "192.168.56.105"; + + @Test + public void case01() throws Throwable { + // given + prepareTable(); + + // when + String[] params = {"-mode", "standalone", "-jobid", "-1", "-job", "src/test/resources/csv2t.json"}; + System.setProperty("datax.home", "../target/datax/datax"); + Engine.entry(params); + } + + public void prepareTable() throws SQLException { + final String url = "jdbc:TAOS-RS://" + host + ":6041"; + try (Connection conn = DriverManager.getConnection(url, "root", "taosdata")) { + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists test"); + stmt.execute("create database if not exists test"); + stmt.execute("create table test.weather (ts timestamp, temperature bigint, humidity double, is_normal bool) " + + "tags(device_id binary(10),address nchar(10))"); + } + } + + +} diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java index 3657a4f..8840aa2 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java @@ -4,6 +4,8 @@ import com.alibaba.datax.common.element.DateColumn; import com.alibaba.datax.common.element.LongColumn; import com.alibaba.datax.common.element.Record; import com.alibaba.datax.common.element.StringColumn; +import com.alibaba.datax.common.plugin.TaskPluginCollector; +import com.alibaba.datax.common.spi.Writer; import com.alibaba.datax.common.util.Configuration; import com.alibaba.datax.core.transport.record.DefaultRecord; import org.junit.AfterClass; @@ -25,6 +27,8 @@ public class DefaultDataHandlerTest { private static final String host = "192.168.1.93"; private static Connection conn; + private final TaskPluginCollector taskPluginCollector = new TDengineWriter.Task().getTaskPluginCollector(); + @Test public void writeSupTableBySQL() throws SQLException { // given @@ -48,8 +52,9 @@ public class DefaultDataHandlerTest { return record; }).collect(Collectors.toList()); + // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); + DefaultDataHandler handler = new DefaultDataHandler(configuration, taskPluginCollector); List tables = configuration.getList("table", String.class); SchemaManager schemaManager = new SchemaManager(conn); Map tableMetas = schemaManager.loadTableMeta(tables); @@ -87,7 +92,7 @@ public class DefaultDataHandlerTest { }).collect(Collectors.toList()); // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); + DefaultDataHandler handler = new DefaultDataHandler(configuration, taskPluginCollector); List tables = configuration.getList("table", String.class); SchemaManager schemaManager = new SchemaManager(conn); Map tableMetas = schemaManager.loadTableMeta(tables); @@ -127,7 +132,7 @@ public class DefaultDataHandlerTest { }).collect(Collectors.toList()); // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); + DefaultDataHandler handler = new DefaultDataHandler(configuration, taskPluginCollector); List tables = configuration.getList("table", String.class); SchemaManager schemaManager = new SchemaManager(connection); Map tableMetas = schemaManager.loadTableMeta(tables); @@ -166,7 +171,7 @@ public class DefaultDataHandlerTest { }).collect(Collectors.toList()); // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); + DefaultDataHandler handler = new DefaultDataHandler(configuration, taskPluginCollector); List tables = configuration.getList("table", String.class); SchemaManager schemaManager = new SchemaManager(conn); Map tableMetas = schemaManager.loadTableMeta(tables); @@ -205,7 +210,7 @@ public class DefaultDataHandlerTest { }).collect(Collectors.toList()); // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); + DefaultDataHandler handler = new DefaultDataHandler(configuration, taskPluginCollector); List tables = configuration.getList("table", String.class); SchemaManager schemaManager = new SchemaManager(conn); Map tableMetas = schemaManager.loadTableMeta(tables); @@ -244,7 +249,7 @@ public class DefaultDataHandlerTest { }).collect(Collectors.toList()); // when - DefaultDataHandler handler = new DefaultDataHandler(configuration); + DefaultDataHandler handler = new DefaultDataHandler(configuration, taskPluginCollector); List tables = configuration.getList("table", String.class); SchemaManager schemaManager = new SchemaManager(conn); Map tableMetas = schemaManager.loadTableMeta(tables); @@ -260,7 +265,7 @@ public class DefaultDataHandlerTest { } private void createSupAndSubTable() throws SQLException { - try(Statement stmt = conn.createStatement()){ + try (Statement stmt = conn.createStatement()) { stmt.execute("drop database if exists scm_test"); stmt.execute("create database if not exists scm_test"); stmt.execute("use scm_test"); @@ -275,7 +280,7 @@ public class DefaultDataHandlerTest { } private void createSupTable() throws SQLException { - try (Statement stmt = conn.createStatement()){ + try (Statement stmt = conn.createStatement()) { stmt.execute("drop database if exists scm_test"); stmt.execute("create database if not exists scm_test"); stmt.execute("use scm_test"); diff --git a/tdenginewriter/src/test/resources/csv2t.json b/tdenginewriter/src/test/resources/csv2t.json new file mode 100644 index 0000000..ef5c4d0 --- /dev/null +++ b/tdenginewriter/src/test/resources/csv2t.json @@ -0,0 +1,80 @@ +{ + "job": { + "content": [ + { + "reader": { + "name": "txtfilereader", + "parameter": { + "path": [ + "/Users/yangzy/IdeaProjects/DataX/tdenginewriter/src/test/resources/weather.csv" + ], + "encoding": "UTF-8", + "column": [ + { + "index": 0, + "type": "string" + }, + { + "index": 1, + "type": "date", + "format": "yyy-MM-dd HH:mm:ss.SSS" + }, + { + "index": 2, + "type": "long" + }, + { + "index": 3, + "type": "double" + }, + { + "index": 4, + "type": "long" + }, + { + "index": 5, + "type": "string" + }, + { + "index": 6, + "type": "String" + } + ], + "fieldDelimiter": "," + } + }, + "writer": { + "name": "tdenginewriter", + "parameter": { + "username": "root", + "password": "taosdata", + "column": [ + "tbname", + "ts", + "temperature", + "humidity", + "is_normal", + "device_id", + "address" + ], + "connection": [ + { + "table": [ + "weather" + ], + "jdbcUrl": "jdbc:TAOS-RS://192.168.56.105:6041/test" + } + ], + "batchSize": 100, + "ignoreTagsUnmatched": true + } + } + } + ], + "setting": { + "speed": { + "channel": 1 + } + } + } +} \ No newline at end of file diff --git a/tdenginewriter/src/test/resources/weather.csv b/tdenginewriter/src/test/resources/weather.csv new file mode 100644 index 0000000..21c4a1a --- /dev/null +++ b/tdenginewriter/src/test/resources/weather.csv @@ -0,0 +1,10 @@ +tb1,2022-02-20 04:05:59.255,5,8.591868744,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 04:58:47.068,3,1.489693641,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 06:31:09.408,1,4.026500719,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 08:08:00.336,1,9.606400360,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 08:28:58.053,9,7.872178184,1,abcABC123123,北京朝阳望京 +tb1,2022-02-20 10:23:20.836,9,2.699478524,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 11:09:59.739,7,7.906723716,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 19:08:29.315,1,5.852338895,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 22:10:06.243,10,5.535007901,1,abcABC123,北京朝阳望京 +tb1,2022-02-20 23:52:43.683,10,10.642013185,1,abcABC123,北京朝阳望京 From 81b004fbb4c53d4b425efed204062b145ce57a7e Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 8 Mar 2022 15:55:19 +0800 Subject: [PATCH 109/114] hotfix tdenginereader mistake --- .../java/com/alibaba/datax/plugin/reader/TDengineReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index 35c82be..332ddf3 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -193,6 +193,7 @@ public class TDengineReader extends Reader { sb.append(" and _c0 < '").append(endTime).append("'"); } String sql = sb.toString().trim(); + sqlList.add(sql); } } else { sqlList.addAll(querySql); From ef8459376072b66c252235fb41e326b88105fb45 Mon Sep 17 00:00:00 2001 From: Trafalgar Date: Tue, 22 Mar 2022 10:59:21 +0800 Subject: [PATCH 110/114] Update sqlserverwriter.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正文档中关于 ["*"] 的显示问题， 主要是 * 在markdown语法的转义导致的 --- sqlserverwriter/doc/sqlserverwriter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserverwriter/doc/sqlserverwriter.md b/sqlserverwriter/doc/sqlserverwriter.md index 255834c..cdaf152 100644 --- a/sqlserverwriter/doc/sqlserverwriter.md +++ b/sqlserverwriter/doc/sqlserverwriter.md @@ -127,7 +127,7 @@ SqlServerWriter 通过 DataX 框架获取 Reader 生成的协议数据，根据 * **column** - * 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用*表示, 例如: "column": ["*"] + * 描述：目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。如果要依次写入全部列，使用*表示, 例如: "column": ["\*"] **column配置项必须指定，不能留空！** From 1a7a00c7d1081801e5cfea2659c9bd430fe13d2c Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 May 2022 10:30:08 +0800 Subject: [PATCH 111/114] resolve code review suggestions --- tdenginereader/doc/tdenginereader-CN.md | 21 +-- .../datax/plugin/reader/TDengineReader.java | 28 ++-- .../writer/tdenginewriter/Constants.java | 4 +- .../tdenginewriter/DefaultDataHandler.java | 157 ++++++++++++------ .../writer/tdenginewriter/SchemaManager.java | 11 +- .../TDengineWriterErrorCode.java | 8 +- .../DefaultDataHandlerTest.java | 12 +- 7 files changed, 141 insertions(+), 100 deletions(-) diff --git a/tdenginereader/doc/tdenginereader-CN.md b/tdenginereader/doc/tdenginereader-CN.md index aa3751e..e950275 100644 --- a/tdenginereader/doc/tdenginereader-CN.md +++ b/tdenginereader/doc/tdenginereader-CN.md @@ -22,7 +22,7 @@ TDengineReader 通过 TDengine 的 JDBC driver 查询获取数据。 "reader": { "name": "tdenginereader", "parameter": { - "user": "root", + "username": "root", "password": "taosdata", "connection": [ { @@ -165,24 +165,8 @@ TDengineReader 通过 TDengine 的 JDBC driver 查询获取数据。 #### 4.1.1 数据特征 -建表语句： - -单行记录类似于： - #### 4.1.2 机器参数 -* 执行DataX的机器参数为: - 1. cpu: - 2. mem: - 3. net: 千兆双网卡 - 4. disc: DataX 数据不落磁盘，不统计此项 - -* TDengine数据库机器参数为: - 1. cpu: - 2. mem: - 3. net: 千兆双网卡 - 4. disc: - #### 4.1.3 DataX jvm 参数 -Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError @@ -201,9 +185,6 @@ TDengineReader 通过 TDengine 的 JDBC driver 查询获取数据。 说明： -1. 这里的单表，主键类型为 bigint(20),自增。 -2. batchSize 和 通道个数，对性能影响较大。 - #### 4.2.4 性能测试小结 1. diff --git a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java index 332ddf3..3797779 100644 --- a/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java +++ b/tdenginereader/src/main/java/com/alibaba/datax/plugin/reader/TDengineReader.java @@ -93,7 +93,7 @@ public class TDengineReader extends Reader { } if (start >= end) throw DataXException.asDataXException(TDengineReaderErrorCode.ILLEGAL_VALUE, - "The parameter [" + Key.BEGIN_DATETIME + "] should be less than the parameter [" + Key.END_DATETIME + "]."); + "The parameter " + Key.BEGIN_DATETIME + ": " + beginDatetime + " should be less than the parameter " + Key.END_DATETIME + ": " + endDatetime + "."); } @@ -119,7 +119,6 @@ public class TDengineReader extends Reader { } } - LOG.info("Configuration: {}", configurations); return configurations; } } @@ -142,15 +141,14 @@ public class TDengineReader extends Reader { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); + } catch (ClassNotFoundException ignored) { + LOG.warn(ignored.getMessage(), ignored); } } @Override public void init() { this.readerSliceConfig = super.getPluginJobConf(); - LOG.info("getPluginJobConf: {}", readerSliceConfig); String user = readerSliceConfig.getString(Key.USERNAME); String password = readerSliceConfig.getString(Key.PASSWORD); @@ -174,7 +172,12 @@ public class TDengineReader extends Reader { @Override public void destroy() { - + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + LOG.error(e.getMessage(), e); + } } @Override @@ -199,22 +202,15 @@ public class TDengineReader extends Reader { sqlList.addAll(querySql); } - try (Statement stmt = conn.createStatement()) { - for (String sql : sqlList) { + for (String sql : sqlList) { + try (Statement stmt = conn.createStatement()) { ResultSet rs = stmt.executeQuery(sql); while (rs.next()) { Record record = buildRecord(recordSender, rs, mandatoryEncoding); recordSender.sendToWriter(record); } - } - } catch (SQLException e) { - throw DataXException.asDataXException(TDengineReaderErrorCode.RUNTIME_EXCEPTION, e.getMessage(), e); - } finally { - try { - if (conn != null) - conn.close(); } catch (SQLException e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); } } } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java index d62c8f3..e044521 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/Constants.java @@ -1,8 +1,6 @@ package com.alibaba.datax.plugin.writer.tdenginewriter; public class Constants { - public static final String DEFAULT_USERNAME = "root"; - public static final String DEFAULT_PASSWORD = "taosdata"; - public static final int DEFAULT_BATCH_SIZE = 1; + public static final int DEFAULT_BATCH_SIZE = 1000; public static final boolean DEFAULT_IGNORE_TAGS_UNMATCHED = false; } \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java index 4ee91ce..9801a32 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandler.java @@ -52,14 +52,13 @@ public class DefaultDataHandler implements DataHandler { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); + } catch (ClassNotFoundException ignored) { } } public DefaultDataHandler(Configuration configuration, TaskPluginCollector taskPluginCollector) { - this.username = configuration.getString(Key.USERNAME, Constants.DEFAULT_USERNAME); - this.password = configuration.getString(Key.PASSWORD, Constants.DEFAULT_PASSWORD); + this.username = configuration.getString(Key.USERNAME); + this.password = configuration.getString(Key.PASSWORD); this.jdbcUrl = configuration.getString(Key.JDBC_URL); this.batchSize = configuration.getInt(Key.BATCH_SIZE, Constants.DEFAULT_BATCH_SIZE); this.tables = configuration.getList(Key.TABLE, String.class); @@ -73,14 +72,15 @@ public class DefaultDataHandler implements DataHandler { int count = 0; int affectedRows = 0; - try (Connection conn = DriverManager.getConnection(jdbcUrl, username, password)) { LOG.info("connection[ jdbcUrl: " + jdbcUrl + ", username: " + username + "] established."); - // prepare table_name -> table_meta - this.schemaManager = new SchemaManager(conn); - this.tableMetas = schemaManager.loadTableMeta(tables); - // prepare table_name -> column_meta - this.columnMetas = schemaManager.loadColumnMetas(tables); + if (schemaManager == null) { + // prepare table_name -> table_meta + this.schemaManager = new SchemaManager(conn); + this.tableMetas = schemaManager.loadTableMeta(tables); + // prepare table_name -> column_meta + this.columnMetas = schemaManager.loadColumnMetas(tables); + } List recordBatch = new ArrayList<>(); Record record; @@ -89,10 +89,11 @@ public class DefaultDataHandler implements DataHandler { recordBatch.add(record); } else { try { - affectedRows = writeBatch(conn, recordBatch); - } catch (SQLException e) { + recordBatch.add(record); + affectedRows += writeBatch(conn, recordBatch); + } catch (Exception e) { LOG.warn("use one row insert. because:" + e.getMessage()); - affectedRows = writeEachRow(conn, recordBatch); + affectedRows += writeEachRow(conn, recordBatch); } recordBatch.clear(); } @@ -101,10 +102,10 @@ public class DefaultDataHandler implements DataHandler { if (!recordBatch.isEmpty()) { try { - affectedRows = writeBatch(conn, recordBatch); - } catch (SQLException e) { + affectedRows += writeBatch(conn, recordBatch); + } catch (Exception e) { LOG.warn("use one row insert. because:" + e.getMessage()); - affectedRows = writeEachRow(conn, recordBatch); + affectedRows += writeEachRow(conn, recordBatch); } recordBatch.clear(); } @@ -126,8 +127,8 @@ public class DefaultDataHandler implements DataHandler { recordList.add(record); try { affectedRows += writeBatch(conn, recordList); - } catch (SQLException e) { - LOG.error(e.getMessage()); + } catch (Exception e) { + LOG.error(e.getMessage(), e); this.taskPluginCollector.collectDirtyRecord(record, e); } } @@ -145,7 +146,7 @@ public class DefaultDataHandler implements DataHandler { * 3. 对于tb，拼sql，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, t1] => insert into tb(ts, f1, f2) values(ts, f1, f2) * 4. 对于t，拼sql，例如：data: [ts, f1, f2, f3, t1, t2] tbColumn: [ts, f1, f2, f3, t1, t2] insert into t(ts, f1, f2, f3, t1, t2) values(ts, f1, f2, f3, t1, t2) */ - public int writeBatch(Connection conn, List recordBatch) throws SQLException { + public int writeBatch(Connection conn, List recordBatch) throws Exception { int affectedRows = 0; for (String table : tables) { TableMeta tableMeta = tableMetas.get(table); @@ -173,31 +174,62 @@ public class DefaultDataHandler implements DataHandler { * record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) * record[idx(tbname)] using table tags(record[idx(t1)]) (ts, f1, f2, f3) values(record[idx(ts)], record[idx(f1)], ) */ - private int writeBatchToSupTableBySQL(Connection conn, String table, List recordBatch) throws SQLException { + private int writeBatchToSupTableBySQL(Connection conn, String table, List recordBatch) throws Exception { List columnMetas = this.columnMetas.get(table); StringBuilder sb = new StringBuilder("insert into"); for (Record record : recordBatch) { sb.append(" ").append(record.getColumn(indexOf("tbname")).asString()) .append(" using ").append(table) - .append(" tags") - .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { - return colMeta.isTag; - }).map(colMeta -> { - return buildColumnValue(colMeta, record); - }).collect(Collectors.joining(",", "(", ")"))) - .append(" ") - .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { + .append(" tags"); +// sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { +// return colMeta.isTag; +// }).map(colMeta -> { +// return buildColumnValue(colMeta, record); +// }).collect(Collectors.joining(",", "(", ")"))); + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + ColumnMeta colMeta = columnMetas.get(i); + if (!columns.contains(colMeta.field)) + continue; + if (!colMeta.isTag) + continue; + String tagValue = buildColumnValue(colMeta, record); + if (i == 0) { + sb.append(tagValue); + } else { + sb.append(",").append(tagValue); + } + } + sb.append(")"); + + sb.append(" ").append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { return !colMeta.isTag; }).map(colMeta -> { return colMeta.field; }).collect(Collectors.joining(",", "(", ")"))) - .append(" values") - .append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { - return !colMeta.isTag; - }).map(colMeta -> { - return buildColumnValue(colMeta, record); - }).collect(Collectors.joining(",", "(", ")"))); + .append(" values"); + +// sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { +// return !colMeta.isTag; +// }).map(colMeta -> { +// return buildColumnValue(colMeta, record); +// }).collect(Collectors.joining(",", "(", ")"))); + sb.append("("); + for (int i = 0; i < columnMetas.size(); i++) { + ColumnMeta colMeta = columnMetas.get(i); + if (!columns.contains(colMeta.field)) + continue; + if (colMeta.isTag) + continue; + String colValue = buildColumnValue(colMeta, record); + if (i == 0) { + sb.append(colValue); + } else { + sb.append(",").append(colValue); + } + } + sb.append(")"); } String sql = sb.toString(); @@ -213,10 +245,11 @@ public class DefaultDataHandler implements DataHandler { return count; } - private String buildColumnValue(ColumnMeta colMeta, Record record) { + private String buildColumnValue(ColumnMeta colMeta, Record record) throws Exception { Column column = record.getColumn(indexOf(colMeta.field)); TimestampPrecision timestampPrecision = schemaManager.loadDatabasePrecision(); - switch (column.getType()) { + Column.Type type = column.getType(); + switch (type) { case DATE: { Date value = column.asDate(); switch (timestampPrecision) { @@ -243,8 +276,9 @@ public class DefaultDataHandler implements DataHandler { case DOUBLE: case INT: case LONG: + column.asString(); default: - return column.asString(); + throw new Exception("invalid column type: " + type); } } @@ -392,7 +426,7 @@ public class DefaultDataHandler implements DataHandler { * else * insert into tb1 (ts, f1, f2) values( record[idx(ts)], record[idx(f1)], record[idx(f2)]) */ - private int writeBatchToSubTable(Connection conn, String table, List recordBatch) throws SQLException { + private int writeBatchToSubTable(Connection conn, String table, List recordBatch) throws Exception { List columnMetas = this.columnMetas.get(table); StringBuilder sb = new StringBuilder(); @@ -419,11 +453,25 @@ public class DefaultDataHandler implements DataHandler { if (ignoreTagsUnmatched && !tagsAllMatch) continue; - sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { - return !colMeta.isTag; - }).map(colMeta -> { - return buildColumnValue(colMeta, record); - }).collect(Collectors.joining(", ", "(", ") "))); +// sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).filter(colMeta -> { +// return !colMeta.isTag; +// }).map(colMeta -> { +// return buildColumnValue(colMeta, record); +// }).collect(Collectors.joining(", ", "(", ") "))); + sb.append("("); + for (int i = 0; i < columnMetas.size(); i++) { + ColumnMeta colMeta = columnMetas.get(i); + if (colMeta.isTag) + continue; + String colValue = buildColumnValue(colMeta, record); + if (i == 0) { + sb.append(colValue); + } else { + sb.append(",").append(colValue); + } + } + sb.append(")"); + validRecords++; } @@ -462,7 +510,7 @@ public class DefaultDataHandler implements DataHandler { * table: ["weather"], column: ["ts, f1, f2, f3, t1, t2"] * sql: insert into weather (ts, f1, f2, f3, t1, t2) values( record[idx(ts), record[idx(f1)], ...) */ - private int writeBatchToNormalTable(Connection conn, String table, List recordBatch) throws SQLException { + private int writeBatchToNormalTable(Connection conn, String table, List recordBatch) throws Exception { List columnMetas = this.columnMetas.get(table); StringBuilder sb = new StringBuilder(); @@ -474,9 +522,22 @@ public class DefaultDataHandler implements DataHandler { .append(" values "); for (Record record : recordBatch) { - sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).map(colMeta -> { - return buildColumnValue(colMeta, record); - }).collect(Collectors.joining(",", "(", ")"))); +// sb.append(columnMetas.stream().filter(colMeta -> columns.contains(colMeta.field)).map(colMeta -> { +// return buildColumnValue(colMeta, record); +// }).collect(Collectors.joining(",", "(", ")"))); + sb.append("("); + for (int i = 0; i < columnMetas.size(); i++) { + ColumnMeta colMeta = columnMetas.get(i); + if (!columns.contains(colMeta.field)) + continue; + String colValue = buildColumnValue(colMeta, record); + if (i == 0) { + sb.append(colValue); + } else { + sb.append(",").append(colValue); + } + } + sb.append(")"); } String sql = sb.toString(); @@ -492,4 +553,4 @@ public class DefaultDataHandler implements DataHandler { "cannot find col: " + colName + " in columns: " + columns); } -} +} \ No newline at end of file diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java index c48b794..f3bdbfb 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/SchemaManager.java @@ -10,6 +10,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.*; +import java.util.stream.Collectors; public class SchemaManager { private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class); @@ -123,12 +124,14 @@ public class SchemaManager { } } } catch (SQLException e) { - e.printStackTrace(); + LOG.error(e.getMessage(), e); } colMeta.value = value; }); - LOG.debug("load column metadata of " + table + ": " + Arrays.toString(columnMetaList.toArray())); + LOG.debug("load column metadata of " + table + ": " + + columnMetaList.stream().map(ColumnMeta::toString).collect(Collectors.joining(",", "[", "]")) + ); ret.put(table, columnMetaList); } return ret; @@ -142,7 +145,9 @@ public class SchemaManager { tableMeta.tags = rs.getInt("tags"); tableMeta.tables = rs.getInt("tables"); - LOG.debug("load table metadata of " + tableMeta.tbname + ": " + tableMeta); + if (LOG.isDebugEnabled()){ + LOG.debug("load table metadata of " + tableMeta.tbname + ": " + tableMeta); + } return tableMeta; } diff --git a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java index 469449e..e1f2bc2 100644 --- a/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java +++ b/tdenginewriter/src/main/java/com/alibaba/datax/plugin/writer/tdenginewriter/TDengineWriterErrorCode.java @@ -4,10 +4,10 @@ import com.alibaba.datax.common.spi.ErrorCode; public enum TDengineWriterErrorCode implements ErrorCode { - REQUIRED_VALUE("TDengineWriter-00", "缺失必要的值"), - ILLEGAL_VALUE("TDengineWriter-01", "值非法"), - RUNTIME_EXCEPTION("TDengineWriter-02", "运行时异常"), - TYPE_ERROR("TDengineWriter-03", "Datax类型无法正确映射到TDengine类型"); + REQUIRED_VALUE("TDengineWriter-00", "parameter value is missing"), + ILLEGAL_VALUE("TDengineWriter-01", "invalid parameter value"), + RUNTIME_EXCEPTION("TDengineWriter-02", "runtime exception"), + TYPE_ERROR("TDengineWriter-03", "data type mapping error"); private final String code; private final String description; diff --git a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java index e0acacb..1034b74 100644 --- a/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java +++ b/tdenginewriter/src/test/java/com/alibaba/datax/plugin/writer/tdenginewriter/DefaultDataHandlerTest.java @@ -28,7 +28,7 @@ public class DefaultDataHandlerTest { private final TaskPluginCollector taskPluginCollector = new TDengineWriter.Task().getTaskPluginCollector(); @Test - public void writeSupTableBySQL() throws SQLException { + public void writeSupTableBySQL() throws Exception { // given createSupAndSubTable(); Configuration configuration = Configuration.from("{" + @@ -68,7 +68,7 @@ public class DefaultDataHandlerTest { } @Test - public void writeSupTableBySQL_2() throws SQLException { + public void writeSupTableBySQL_2() throws Exception { // given createSupAndSubTable(); Configuration configuration = Configuration.from("{" + @@ -106,7 +106,7 @@ public class DefaultDataHandlerTest { } @Test - public void writeSupTableBySchemaless() throws SQLException { + public void writeSupTableBySchemaless() throws Exception { // given createSupTable(); Configuration configuration = Configuration.from("{" + @@ -146,7 +146,7 @@ public class DefaultDataHandlerTest { } @Test - public void writeSubTableWithTableName() throws SQLException { + public void writeSubTableWithTableName() throws Exception { // given createSupAndSubTable(); Configuration configuration = Configuration.from("{" + @@ -185,7 +185,7 @@ public class DefaultDataHandlerTest { } @Test - public void writeSubTableWithoutTableName() throws SQLException { + public void writeSubTableWithoutTableName() throws Exception { // given createSupAndSubTable(); Configuration configuration = Configuration.from("{" + @@ -224,7 +224,7 @@ public class DefaultDataHandlerTest { } @Test - public void writeNormalTable() throws SQLException { + public void writeNormalTable() throws Exception { // given createSupAndSubTable(); Configuration configuration = Configuration.from("{" + From f0fcca0810b504d3057d1b96f409c6e90b65ad4c Mon Sep 17 00:00:00 2001 From: JohnnyJie <771886728@qq.com> Date: Thu, 12 May 2022 11:23:22 +0800 Subject: [PATCH 112/114] Update tsdbhttpwriter.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TSDB Writer文档 --- tsdbwriter/doc/tsdbhttpwriter.md | 152 ++++++++++++++++++++++++++++--- 1 file changed, 138 insertions(+), 14 deletions(-) diff --git a/tsdbwriter/doc/tsdbhttpwriter.md b/tsdbwriter/doc/tsdbhttpwriter.md index c723a36..87f4d13 100644 --- a/tsdbwriter/doc/tsdbhttpwriter.md +++ b/tsdbwriter/doc/tsdbhttpwriter.md @@ -6,24 +6,57 @@ ___ ## 1 快速介绍 -TSDBWriter 插件实现了将数据点写入到阿里巴巴自研 TSDB 数据库中（后续简称 TSDB）。 +TSDBWriter 插件实现了将数据点写入到阿里巴巴云原生多模数据库Lindorm TSDB数据库中（后续简称 TSDB）。 -时间序列数据库（Time Series Database , 简称 TSDB）是一种高性能，低成本，稳定可靠的在线时序数据库服务；提供高效读写，高压缩比存储、时序数据插值及聚合计算，广泛应用于物联网（IoT）设备监控系统 ，企业能源管理系统（EMS），生产安全监控系统，电力检测系统等行业场景。 TSDB 提供百万级时序数据秒级写入，高压缩比低成本存储、预降采样、插值、多维聚合计算，查询结果可视化功能；解决由于设备采集点数量巨大，数据采集频率高，造成的存储成本高，写入和查询分析效率低的问题。更多关于 TSDB 的介绍，详见[阿里云 TSDB 官网](https://help.aliyun.com/product/54825.html)。 +时间序列数据库（Time Series Database , 简称 TSDB）是一种高性能，低成本，稳定可靠的在线时序数据库服务；提供高效读写，高压缩比存储、时序数据插值及聚合计算，广泛应用于物联网（IoT）设备监控系统 ，企业能源管理系统（EMS），生产安全监控系统，电力检测系统等行业场景。 TSDB 提供千万级时序数据秒级写入，高压缩比低成本存储、预降采样、插值、多维聚合计算，查询结果可视化功能；解决由于设备采集点数量巨大，数据采集频率高，造成的存储成本高，写入和查询分析效率低的问题。更多关于 TSDB 的介绍，详见[阿里云 Lindorm TSDB 官网](https://help.aliyun.com/document_detail/174600.html)。 +注意：阿里巴巴自研HiTSDB已全新升级为云原生多模数据库Lindorm TSDB。Lindorm TSDB兼容大部分HiTSDB的HTTP API并提供原生SQL能力，TSDBWriter插件使用HTTP API方式写入，要使用原生SQL能力需要提前在Lindorm TSDB进行建表。详细参见[与旧版TSDB的比较](https://help.aliyun.com/document_detail/387477.html) ## 2 实现原理 -通过 HTTP 连接 TSDB 实例，并通过 `/api/put` 接口将数据点写入。关于写入接口详见 TSDB 的[接口说明文档](https://help.aliyun.com/document_detail/59939.html)。 - - +通过TSDB客户端 hitsdb-client 连接 TSDB 实例，并将数据点通过HTTP API方式写入。关于写入接口详见TSDB 的[SDK 参考](https://help.aliyun.com/document_detail/61587.html)。 ## 3 功能说明 ### 3.1 配置样例 -* 配置一个从 OpenTSDB 数据库同步抽取数据到 TSDB： +* 配置TSDB Writer： +```json +{ + "name": "tsdbwriter", + "parameter": { + "endpoint": "http://localhost:8242", + "sourceDbType": "RDB", + "batchSize": 256, + "columnType": [ + "tag", + "tag", + "field_string", + "field_double", + "timestamp", + "field_boolean" + ], + "column": [ + "tag1", + "tag2", + "field1", + "field2", + "timestamp", + "field3" + ], + "multiField":"true", + "table":"testmetric", + "username":"xxx", + "password":"xxx", + "ignoreWriteError":"false", + "database":"default" + } +} +``` + +* 配置一个从 支持 OpenTSDB 协议的数据库同步抽取数据到 TSDB： ```json { @@ -42,7 +75,7 @@ TSDBWriter 插件实现了将数据点写入到阿里巴巴自研 TSDB 数据库 } }, "writer": { - "name": "tsdbhttpwriter", + "name": "tsdbwriter", "parameter": { "endpoint": "http://localhost:8242" } @@ -58,7 +91,36 @@ TSDBWriter 插件实现了将数据点写入到阿里巴巴自研 TSDB 数据库 } ``` - +* 使用 OpenTSDB （单值）协议写入TSDB（不推荐）： +```json +{ + "name": "tsdbwriter", + "endpoint": "http://localhost:8242", + "sourceDbType": "RDB", + "parameter": { + "batchSize": 256, + "columnType": [ + "tag", + "tag", + "field_string", + "field_double", + "timestamp", + "field_boolean" + ], + "column": [ + "tag1", + "tag2", + "field_metric_1", + "field_metric_2", + "timestamp", + "field_metric_3" + ], + "username":"tsdb", + "password":"enxU^", + "ignoreWriteError":"false" + } +} +``` ### 3.2 参数说明 @@ -74,23 +136,85 @@ TSDBWriter 插件实现了将数据点写入到阿里巴巴自研 TSDB 数据库 * 格式：http://IP:Port * 默认值：无 +* **sourceDbType** + * 描述：源端数据类型 + * 必选：否 + * 格式：string [RDB或者TSDB] + * 默认值：TSDB + +* **multiField** + * 描述：使用HTTP API多值（多个field）方式写入，目前TSDB版本使用多值写入，需要指定为true + * 必选：是 + * 格式：bool + * 默认值：false （单值） + * 说明: 如果使用Lindorm TSDB原生SQL能力访问HTTP API方式写入的数据，需要在TSDB进行预建表，否则只能使用HiTSDB HTTP API方式[查询数据](https://help.aliyun.com/document_detail/107576.html)。 + +* **column** + * 描述：关系型数据库中表的字段名 + * 必选：当sourceDbType为RDB时必选 + * 格式：string + * 默认值：无 + * 说明: 此处的字段顺序，需要和Reader插件中配置的column字段的顺序保持一致。 + +* **columnType** + * 描述：关系型数据库中表字段，映射到TSDB中的类型。支持的类型如下所示： + * timestamp：该字段为时间戳 + * tag：该字段为tag + * field_string: 该Field的value是字符串类型 + * field_double: 该Field的value是数值类型 + * field_boolean: 该Field的value是布尔类型 + * 必选：当sourceDbType为RDB时必选 + * 格式：string [RDB或者TSDB] + * 默认值: 无 + * 说明: 此处的字段顺序，需要和column配置中的字段顺序保持一致 + +* **table** + * 描述：TSDB对应表名（metric） + * 必选：当sourceDbType为RDB时且multiField为true时必选 + * 格式：string + * 默认值：无 + * 说明: 要导入的TSDB表名，如果multiField为falase,不需要填写，对应的metric需要写到column字段 + * **batchSize** * 描述：每次批量数据的条数 * 必选：否 * 格式：int，需要保证大于 0 * 默认值：100 +* **ignoreWriteError** + * 描述：如果设置为 true，则忽略写入错误，继续写入；否则，多次重试后仍写入失败的话，则会终止写入任务 + * 必选：否 + * 格式：bool + * 默认值：false + +* **username** + * 描述：数据库用户名 + * 必选：否 + * 格式：string + * 默认值：无 + * 说明: TSDB配置了鉴权需要填写 + + +* **password** + * 描述：数据库密码 + * 必选：否 + * 格式：string + * 默认值：无 + * 说明: TSDB配置了鉴权需要填写 + +* **database** + * 描述：导入的数据库 + * 必选：否 + * 格式：string + * 默认值：default + * 说明: TSDB需要提前创建数据库 + * **maxRetryTime** * 描述：失败后重试的次数 * 必选：否 * 格式：int，需要保证大于 1 * 默认值：3 -* **ignoreWriteError** - * 描述：如果设置为 true，则忽略写入错误，继续写入；否则，多次重试后仍写入失败的话，则会终止写入任务 - * 必选：否 - * 格式：bool - * 默认值：false @@ -171,7 +295,7 @@ HBase 机型： 8C16G * 5 ## 5 约束限制 -### 5.1 目前只支持兼容 TSDB 2.4.x 及以上版本 +### 5.1 目前支持Lindorm TSDB全部版本 以及 HiTSDB 2.4.x 及以上版本 其他版本暂不保证兼容 From 64eeea95bbcb446efef922e20a4ca87be484c1f4 Mon Sep 17 00:00:00 2001 From: JohnnyJie <771886728@qq.com> Date: Thu, 12 May 2022 11:44:46 +0800 Subject: [PATCH 113/114] Update tsdbhttpwriter.md --- tsdbwriter/doc/tsdbhttpwriter.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsdbwriter/doc/tsdbhttpwriter.md b/tsdbwriter/doc/tsdbhttpwriter.md index 87f4d13..c0b1365 100644 --- a/tsdbwriter/doc/tsdbhttpwriter.md +++ b/tsdbwriter/doc/tsdbhttpwriter.md @@ -36,7 +36,7 @@ TSDBWriter 插件实现了将数据点写入到阿里巴巴云原生多模数据 "field_string", "field_double", "timestamp", - "field_boolean" + "field_bool" ], "column": [ "tag1", @@ -121,6 +121,7 @@ TSDBWriter 插件实现了将数据点写入到阿里巴巴云原生多模数据 } } ``` +转换到的TSDB 表名（metric）由column中field对应的列名决定：对于上诉配置一行关系型数据将会写入三个metric（field_metric_1，field_metric_2，field_metric_3） ### 3.2 参数说明 From 1b20d9a9e53244113266f8f0bcb8a26935df49b1 Mon Sep 17 00:00:00 2001 From: ZhangLu <49864044+VVeaker@users.noreply.github.com> Date: Thu, 19 May 2022 11:44:57 +0800 Subject: [PATCH 114/114] Update mongodbwriter.md --- mongodbwriter/doc/mongodbwriter.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mongodbwriter/doc/mongodbwriter.md b/mongodbwriter/doc/mongodbwriter.md index 74de8a0..b3bfece 100644 --- a/mongodbwriter/doc/mongodbwriter.md +++ b/mongodbwriter/doc/mongodbwriter.md @@ -116,9 +116,9 @@ MongoDBWriter通过Datax框架获取Reader生成的数据，然后将Datax支持 "type": "int" } ], - "upsertInfo": { - "isUpsert": "true", - "upsertKey": "unique_id" + "writeMode": { + "isReplace": "true", + "replaceKey": "unique_id" } } } @@ -135,11 +135,11 @@ MongoDBWriter通过Datax框架获取Reader生成的数据，然后将Datax支持 * collectionName： MonogoDB的集合名。【必填】 * column：MongoDB的文档列名。【必填】 * name：Column的名字。【必填】 -* type：Column的类型。【选填】 +* type：Column的类型。【必填】 * splitter：特殊分隔符，当且仅当要处理的字符串要用分隔符分隔为字符数组时，才使用这个参数，通过这个参数指定的分隔符，将字符串分隔存储到MongoDB的数组中。【选填】 -* upsertInfo：指定了传输数据时更新的信息。【选填】 -* isUpsert：当设置为true时，表示针对相同的upsertKey做更新操作。【选填】 -* upsertKey：upsertKey指定了每行记录的业务主键。用来做更新时使用。【选填】 +* writeMode：指定了传输数据时更新的信息。【选填】 +* isReplace：当设置为true时，表示针对相同的replaceKey做更新操作。【选填】 +* replaceKey：replaceKey指定了每行记录的业务主键。用来做更新时使用。【选填】 #### 5 类型转换