From 23425525828e8ac9e421a1fde1638931ef70f409 Mon Sep 17 00:00:00 2001 From: zcx <48920254+Ceilzcx@users.noreply.github.com> Date: Wed, 26 Apr 2023 20:00:42 +0800 Subject: [PATCH] support influxdb as history data store database (#917) support influxdb as history database support influxdb as history database implement influxdb as history database merge master to here update pom fix timezone update influxdb config and create retention policy update influxdb ssl connect update influxdb prop update influxdb optimize createDatabase method update influxdb getHistoryIntervalMetricData update influxdb [doc] add influxdb doc update application --------- --- home/docs/start/influxdb-init.md | 71 ++++ .../current/start/influxdb-init.md | 73 ++++ .../current/start/mysql-change.md | 4 +- .../current/start/postgresql-change.md | 4 +- home/sidebars.json | 1 + .../manager/service/impl/AppServiceImpl.java | 4 + .../src/main/resources/application-prod.yml | 74 ---- manager/src/main/resources/application.yml | 90 ++++- warehouse/pom.xml | 53 +++ .../warehouse/config/WarehouseProperties.java | 58 ++-- .../store/HistoryInfluxdbDataStorage.java | 316 ++++++++++++++++++ .../store/HistoryTdEngineDataStorage.java | 4 +- .../monitor-data-chart.component.ts | 4 +- 13 files changed, 650 insertions(+), 106 deletions(-) create mode 100644 home/docs/start/influxdb-init.md create mode 100644 home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/influxdb-init.md delete mode 100644 manager/src/main/resources/application-prod.yml create mode 100644 warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryInfluxdbDataStorage.java diff --git a/home/docs/start/influxdb-init.md b/home/docs/start/influxdb-init.md new file mode 100644 index 0000000000..b5a482ac41 --- /dev/null +++ b/home/docs/start/influxdb-init.md @@ -0,0 +1,71 @@ +--- +id: influxdb-init +title: Use Time Series Database InfluxDB to Store Metrics Data (Optional) +sidebar_label: Use InfluxDB Store Metrics +--- + +HertzBeat's historical data storage depends on the time series database InfluxDB, TDengine or IoTDB, choose one of them to install and initialize, or not to install (note ⚠️ but it is strongly recommended to configure in the production environment) + +TDengine is the Time Series Data Platform where developers build IoT, analytics, and cloud applications. + +**Note⚠️ Time series database is optional, but production environment configuration is strongly recommended to provide more complete historical chart functions and high performance** + +Note⚠️ Need InfluxDB 1.x Version. + +### 1. Use HuaweiCloud GaussDB For Influx + +> Use [HuaweiCloud GaussDB For Influx](https://www.huaweicloud.com/product/gaussdbforinflux.html) + +> Get the `GaussDB For Influx` service url, username and password config. + +⚠️Note `GaussDB For Influx` enable SSL default, the service url should use `https:` + +### 2. Install TDengine via Docker +> Refer to the official website [installation tutorial](https://hub.docker.com/_/influxdb) +1. Download and install Docker environment + Docker tools download refer to [Docker official document](https://docs.docker.com/get-docker/). + After the installation you can check if the Docker version normally output at the terminal. + ``` + $ docker -v + Docker version 20.10.12, build e91ed57 + ``` +2. Install InfluxDB with Docker + ``` + $ docker run -p 8086:8086 \ + -v /opt/influxdb:/var/lib/influxdb \ + influxdb:1.8 + ``` + `-v /opt/influxdb:/var/lib/influxdb` is local persistent mount of InfluxDB data directory. `/opt/influxdb` should be replaced with the actual local directory. + use```$ docker ps``` to check if the database started successfully + + +### Configure the database connection in hertzbeat `application.yml` configuration file + +1. Configure HertzBeat's configuration file + Modify `hertzbeat/config/application.yml` configuration file [/script/application.yml](https://github.com/dromara/hertzbeat/raw/master/script/application.yml) + Note⚠️The docker container way need to mount application.yml file locally,while you can use installation package way to unzip and modify `hertzbeat/config/application.yml` + Replace `warehouse.store.influxdb` data source parameters, URL account and password. + +```yaml +warehouse: + store: + # disable jpa + jpa: + enabled: false + # enable influxdb + influxdb: + enabled: true + server-url: http://localhost:8086 + username: root + password: root + expire-time: '30d' + replication: 1 +``` + +2. Restart HertzBeat + +### FAQ + +1. Do both the time series databases InfluxDB, IoTDB and TDengine need to be configured? Can they both be used? + +> You don't need to configure all of them, you can choose one of them. Use the enable parameter to control whether it is used or not. You can also install and configure neither, which only affects the historical chart data. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/influxdb-init.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/influxdb-init.md new file mode 100644 index 0000000000..7389308754 --- /dev/null +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/influxdb-init.md @@ -0,0 +1,73 @@ +--- +id: influxdb-init +title: 依赖时序数据库服务InfluxDB安装初始化 +sidebar_label: 使用InfluxDB存储指标数据(可选) +--- + +HertzBeat的历史数据存储依赖时序数据库 InfluxDB, TDengine 或 IoTDB，任选其一安装初始化即可，也可不安装(注意⚠️但强烈建议生产环境配置) + +InfluxDB是一个由InfluxData开发的开源时序型数据库，专注于海量时序数据的高性能读、高性能写、高效存储与实时分析等。 注意支持⚠️ 1.x版本。 + +**注意⚠️ 时序数据库安装配置为可选项，但强烈建议生产环境配置，以提供更完善的历史图表功能，高性能和稳定性** + +### 1. 直接使用华为云服务 GaussDB For Influx + +> 开通使用[华为云云数据库 GaussDB For Influx](https://www.huaweicloud.com/product/gaussdbforinflux.html) + +> 获取云数据库对外暴露连接地址，账户密码即可 + +⚠️注意云数据库默认开启了SSL，云数据库地址应使用 `https:` + +### 2. 通过Docker方式安装InfluxDB +> 可参考官方网站[安装教程](https://hub.docker.com/_/influxdb) +1. 下载安装Docker环境 + Docker 工具自身的下载请参考 [Docker官网文档](https://docs.docker.com/get-docker/)。 + 安装完毕后终端查看Docker版本是否正常输出。 + ``` + $ docker -v + Docker version 20.10.12, build e91ed57 + ``` +2. Docker安装InfluxDB 1.x + +```shell +$ docker run -p 8086:8086 \ + -v /opt/influxdb:/var/lib/influxdb \ + influxdb:1.8 +``` + + `-v /opt/influxdb:/var/lib/influxdb` 为influxdb数据目录本地持久化挂载，需将`/opt/influxdb`替换为实际本地存在的目录 + 使用```$ docker ps```查看数据库是否启动成功 + + +### 在hertzbeat的`application.yml`配置文件配置此数据库连接 + +1. 配置HertzBeat的配置文件 + 修改位于 `hertzbeat/config/application.yml` 的配置文件 [/script/application.yml](https://github.com/dromara/hertzbeat/raw/master/script/application.yml) + 注意⚠️docker容器方式需要将application.yml文件挂载到主机本地,安装包方式解压修改位于 `hertzbeat/config/application.yml` 即可 + +**修改里面的`warehouse.store.jpa.enabled`参数为`false`， 配置里面的`warehouse.store.influxdb`数据源参数，URL账户密码，并启用`enabled`为`true`** + +```yaml +warehouse: + store: + # 关闭默认JPA + jpa: + enabled: false + influxdb: + enabled: true + server-url: http://localhost:8086 + username: root + password: root + expire-time: '30d' + replication: 1 +``` + +2. 重启 HertzBeat + +### 常见问题 + +1. 时序数据库InfluxDb, IoTDB和TDengine是否都需要配置，能不能都用 + +> 不需要都配置，任选其一即可，用enable参数控制其是否使用，也可都不安装配置，只影响历史图表数据。 + + diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/mysql-change.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/mysql-change.md index c174bbe4a8..a905b81436 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/mysql-change.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/mysql-change.md @@ -1,7 +1,7 @@ --- id: mysql-change -title: 依赖的关系型数据库 H2 切换 为MYSQL -sidebar_label: 使用Mysql替换内置H2数据库(可选) +title: 关系型数据库使用 Mysql 替换依赖的 H2 存储系统元数据 +sidebar_label: 元数据使用Mysql存储(可选) --- MYSQL是一款值得信赖的关系型数据库，HertzBeat除了支持使用默认内置的H2数据库外，还可以切换为使用MYSQL存储监控信息，告警信息，配置信息等结构化关系数据。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/postgresql-change.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/postgresql-change.md index e1f7d572ae..ef349af156 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/postgresql-change.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/start/postgresql-change.md @@ -1,7 +1,7 @@ --- id: postgresql-change -title: 依赖的关系型数据库 H2 切换 为 PostgreSQL -sidebar_label: 使用PostgreSQL替换内置H2数据库(可选) +title: 关系型数据库使用 PostgreSQL 替换依赖的 H2 存储系统元数据 +sidebar_label: 元数据使用PostgreSQL存储(可选) --- PostgreSQL是一个功能强大，开源的关系型数据库管理系统（RDBMS）。HertzBeat除了支持使用默认内置的H2数据库外，还可以切换为使用PostgreSQL存储监控信息，告警信息，配置信息等结构化关系数据。 diff --git a/home/sidebars.json b/home/sidebars.json index 7e3d2225c4..7e03183171 100644 --- a/home/sidebars.json +++ b/home/sidebars.json @@ -12,6 +12,7 @@ "start/iotdb-init", "start/tdengine-init", "start/greptime-init", + "start/influxdb-init", "start/mysql-change", "start/postgresql-change", "start/account-modify", diff --git a/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java b/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java index 6d9736846f..c5535defec 100644 --- a/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java +++ b/manager/src/main/java/org/dromara/hertzbeat/manager/service/impl/AppServiceImpl.java @@ -245,7 +245,11 @@ public class AppServiceImpl implements AppService, CommandLineRunner { Assert.notNull(app.getCategory(), "define yml require attributes category"); Assert.notEmpty(app.getName(), "define yml require attributes name"); Assert.notEmpty(app.getParams(), "define yml require attributes params"); + boolean hasParamHost = app.getParams().stream().anyMatch(item -> "host".equals(item.getField())); + Assert.isTrue(hasParamHost, "define yml attributes params must have param host"); Assert.notEmpty(app.getMetrics(), "define yml require attributes metrics"); + boolean hasAvailableMetrics = app.getMetrics().stream().anyMatch(item -> item.getPriority() == 0); + Assert.isTrue(hasAvailableMetrics, "define yml metrics list must have one priority 0 metrics"); } @Override diff --git a/manager/src/main/resources/application-prod.yml b/manager/src/main/resources/application-prod.yml deleted file mode 100644 index ae2a0a5fad..0000000000 --- a/manager/src/main/resources/application-prod.yml +++ /dev/null @@ -1,74 +0,0 @@ -spring: - datasource: - driver-class-name: org.h2.Driver - username: sa - password: 123456 - url: jdbc:h2:./data/hertzbeat;MODE=MYSQL - hikari: - max-lifetime: 120000 - - jpa: - hibernate: - ddl-auto: update - - # Not Require, Please config if you need email notify - # 非必填：不使用邮箱作为警告通知可以去掉spring.mail配置 - mail: - # Attention: this is mail server address. - # 请注意此为邮件服务器地址：qq邮箱为 smtp.qq.com qq企业邮箱为 smtp.exmail.qq.com - host: smtp.qq.com - username: example@tancloud.cn - # Attention: this is not email account password, this requires an email authorization code - # 请注意此非邮箱账户密码 此需填写邮箱授权码 - password: example - port: 465 - default-encoding: UTF-8 - properties: - mail: - smtp: - socketFactoryClass: javax.net.ssl.SSLSocketFactory - ssl: - enable: true - debug: false - -warehouse: - store: - # store history metrics data, enable only one below - # 存储历史数据方式, 下方只能enabled启用一种方式 - jpa: - enabled: true - expire-time: 1h - td-engine: - enabled: false - driver-class-name: com.taosdata.jdbc.rs.RestfulDriver - url: jdbc:TAOS-RS://localhost:6041/hertzbeat - username: root - password: taosdata - iot-db: - enabled: false - host: 127.0.0.1 - rpc-port: 6667 - username: root - password: root - # org.dromara.hertzbeat.warehouse.config.IotDbVersion: V_0_13 || V_1_0 - version: V_0_13 - query-timeout-in-ms: -1 - # 数据存储时间：默认'7776000000'（90天,单位为毫秒,-1代表永不过期） - # data expire time, unit:ms, default '7776000000'(90 days, -1:never expire) - expire-time: '7776000000' - # store real-time metrics data, enable only one below - # 存储实时数据方式, 下方只能enabled启用一种方式 - memory: - enabled: true - init-size: 1024 - redis: - enabled: false - host: 127.0.0.1 - port: 6379 - password: 123456 - #redis使用数据库，默认为DB0 - db: 0 - -alerter: - # custom console url - console-url: https://console.tancloud.cn \ No newline at end of file diff --git a/manager/src/main/resources/application.yml b/manager/src/main/resources/application.yml index 478976f9f7..312409bc3d 100644 --- a/manager/src/main/resources/application.yml +++ b/manager/src/main/resources/application.yml @@ -61,4 +61,92 @@ sureness: secret: 'CyaFv0bwq2Eik0jdrKUtsA6bx3sDJeFV643R LnfKefTjsIfJLBa2YkhEqEGtcHDTNe4CU6+9 8tVt4bisXQ13rbN0oxhUZR73M6EByXIO+SV5 - dKhaX0csgOCTlCxq20yhmUea6H6JIpSE2Rwp' \ No newline at end of file + dKhaX0csgOCTlCxq20yhmUea6H6JIpSE2Rwp' + +--- +spring: + config: + activate: + on-profile: prod + + datasource: + driver-class-name: org.h2.Driver + username: sa + password: 123456 + url: jdbc:h2:./data/hertzbeat;MODE=MYSQL + hikari: + max-lifetime: 120000 + + jpa: + hibernate: + ddl-auto: update + + # Not Require, Please config if you need email notify + # 非必填：不使用邮箱作为警告通知可以去掉spring.mail配置 + mail: + # Attention: this is mail server address. + # 请注意此为邮件服务器地址：qq邮箱为 smtp.qq.com qq企业邮箱为 smtp.exmail.qq.com + host: smtp.qq.com + username: example@tancloud.cn + # Attention: this is not email account password, this requires an email authorization code + # 请注意此非邮箱账户密码 此需填写邮箱授权码 + password: example + port: 465 + default-encoding: UTF-8 + properties: + mail: + smtp: + socketFactoryClass: javax.net.ssl.SSLSocketFactory + ssl: + enable: true + debug: false + +warehouse: + store: + # store history metrics data, enable only one below + # 存储历史数据方式, 下方只能enabled启用一种方式 + jpa: + enabled: true + expire-time: 1h + td-engine: + enabled: false + driver-class-name: com.taosdata.jdbc.rs.RestfulDriver + url: jdbc:TAOS-RS://localhost:6041/hertzbeat + username: root + password: taosdata + iot-db: + enabled: false + host: 127.0.0.1 + rpc-port: 6667 + username: root + password: root + # org.dromara.hertzbeat.warehouse.config.IotDbVersion: V_0_13 || V_1_0 + version: V_0_13 + query-timeout-in-ms: -1 + # 数据存储时间：默认'7776000000'（90天,单位为毫秒,-1代表永不过期） + # data expire time, unit:ms, default '7776000000'(90 days, -1:never expire) + expire-time: '7776000000' + influxdb: + enabled: false + server-url: http://127.0.0.1:8086 + username: root + password: root + expire-time: '30d' + replication: 1 + + # store real-time metrics data, enable only one below + # 存储实时数据方式, 下方只能enabled启用一种方式 + memory: + enabled: true + init-size: 1024 + redis: + enabled: false + host: 127.0.0.1 + port: 6379 + password: 123456 + #redis使用数据库，默认为DB0 + db: 0 + +alerter: + # custom console url + console-url: https://console.tancloud.cn diff --git a/warehouse/pom.xml b/warehouse/pom.xml index 515a878a36..309686fc10 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -24,6 +24,12 @@ 1.0 + 1.0 + 0.13.3 + 2.23 + 3.0.5 + 3.0.0 + 0.1.3 4.0.0 @@ -66,19 +72,66 @@ com.taosdata.jdbc taos-jdbcdriver + ${taos-jdbcdriver.version} org.apache.iotdb iotdb-session + ${iotdb-session.version} + + + org.influxdb + influxdb-java + ${influxdb.version} + + io.greptime greptimedb-protocol + ${greptimedb.version} + + + io.grpc + grpc-all + + + com.google.code.gson + gson + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + io.greptime greptimedb-grpc + ${greptimedb.version} + + + io.grpc + grpc-all + + + com.google.code.gson + gson + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java index 2d74a20b02..91c206650e 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/config/WarehouseProperties.java @@ -284,19 +284,23 @@ public class WarehouseProperties { /** * influxdb的连接服务器url */ - private String servers = "http://127.0.0.1:8086"; + private String serverUrl; /** - * 认证token + * 用户名 */ - private String token; + private String username; /** - * 仓库名称 + * 密码 */ - private String bucket; + private String password; /** - * 组织名称 + * 过期时间 */ - private String org; + private String expireTime = "30d"; + /** + * 副本数 + */ + private int replication = 1; public boolean isEnabled() { return enabled; @@ -306,36 +310,44 @@ public class WarehouseProperties { this.enabled = enabled; } - public String getServers() { - return servers; + public String getServerUrl() { + return serverUrl; } - public void setServers(String servers) { - this.servers = servers; + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; } - public String getToken() { - return token; + public String getUsername() { + return username; } - public void setToken(String token) { - this.token = token; + public void setUsername(String username) { + this.username = username; } - public String getBucket() { - return bucket; + public String getPassword() { + return password; } - public void setBucket(String bucket) { - this.bucket = bucket; + public void setPassword(String password) { + this.password = password; } - public String getOrg() { - return org; + public String getExpireTime() { + return expireTime; } - public void setOrg(String org) { - this.org = org; + public void setExpireTime(String expireTime) { + this.expireTime = expireTime; + } + + public int getReplication() { + return replication; + } + + public void setReplication(int replication) { + this.replication = replication; } } diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryInfluxdbDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryInfluxdbDataStorage.java new file mode 100644 index 0000000000..cb778294e0 --- /dev/null +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryInfluxdbDataStorage.java @@ -0,0 +1,316 @@ +package org.dromara.hertzbeat.warehouse.store; + +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import org.apache.http.ssl.SSLContexts; +import org.dromara.hertzbeat.common.constants.CommonConstants; +import org.dromara.hertzbeat.common.entity.dto.Value; +import org.dromara.hertzbeat.common.entity.message.CollectRep; +import org.dromara.hertzbeat.warehouse.config.WarehouseProperties; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.BatchPoints; +import org.influxdb.dto.Point; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import javax.net.ssl.*; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * + * @since 7/4/2023 + */ +@Component +@ConditionalOnProperty(prefix = "warehouse.store.influxdb", + name = "enabled", havingValue = "true") +@Slf4j +public class HistoryInfluxdbDataStorage extends AbstractHistoryDataStorage { + + private static final String DATABASE = "hertzbeat"; + + private static final String SHOW_DATABASE = "SHOW DATABASES"; + + private static final String CREATE_DATABASE = "CREATE DATABASE %s"; + + private static final String QUERY_HISTORY_SQL = "SELECT instance, %s FROM %s WHERE time >= now() - %s order by time desc"; + + private static final String QUERY_HISTORY_SQL_WITH_INSTANCE = "SELECT instance, %s FROM %s WHERE instance = '%s' and time >= now() - %s order by time desc"; + + private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL = + "SELECT FIRST(%s), MEAN(%s), MAX(%s), MIN(%s) FROM %s WHERE instance = '%s' and time >= now() - %s GROUP BY time(4h)"; + + private static final String CREATE_RETENTION_POLICY = "CREATE RETENTION POLICY \"%s_retention\" ON \"%s\" DURATION %s REPLICATION %d DEFAULT"; + + private static final String QUERY_INSTANCE_SQL = "show tag values from %s with key = \"instance\""; + + private InfluxDB influxDb; + + public HistoryInfluxdbDataStorage(WarehouseProperties properties) { + this.initInfluxDb(properties); + } + + public void initInfluxDb(WarehouseProperties properties) { + OkHttpClient.Builder client = new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .writeTimeout(10, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .retryOnConnectionFailure(true); + + client.sslSocketFactory(defaultSslSocketFactory(), defaultTrustManager()); + client.hostnameVerifier(noopHostnameVerifier()); + + WarehouseProperties.StoreProperties.InfluxdbProperties influxdbProperties = properties.getStore().getInfluxdb(); + this.influxDb = InfluxDBFactory.connect(influxdbProperties.getServerUrl(), influxdbProperties.getUsername(), influxdbProperties.getPassword(), client); + // Close it if your application is terminating, or you are not using it anymore. + Runtime.getRuntime().addShutdownHook(new Thread(influxDb::close)); + + this.serverAvailable = this.createDatabase(influxdbProperties); + } + + private boolean createDatabase(WarehouseProperties.StoreProperties.InfluxdbProperties influxdbProperties) { + QueryResult queryResult = this.influxDb.query(new Query(SHOW_DATABASE)); + + if (queryResult.hasError()) { + log.error("show databases in influxdb error, msg: {}", queryResult.getError()); + return false; + } + + for (QueryResult.Result result : queryResult.getResults()) { + for (QueryResult.Series series : result.getSeries()) { + for (List values : series.getValues()) { + if (values.contains(DATABASE)) { + // database exists + return true; + } + } + } + } + + // 创建数据库 + String createDatabaseSql = String.format(CREATE_DATABASE, DATABASE); + QueryResult createDatabaseResult = this.influxDb.query(new Query(createDatabaseSql)); + if (createDatabaseResult.hasError()) { + log.error("create database {} in influxdb error, msg: {}", DATABASE, createDatabaseResult.getError()); + return false; + } + // 设置过期时间 + String createRetentionPolicySql = String.format(CREATE_RETENTION_POLICY, DATABASE, DATABASE, + influxdbProperties.getExpireTime(), influxdbProperties.getReplication()); + QueryResult createRetentionPolicySqlResult = this.influxDb.query(new Query(createRetentionPolicySql)); + if (createRetentionPolicySqlResult.hasError()) { + log.error("create retention policy in influxdb error, msg: {}", createDatabaseResult.getError()); + return false; + } + + return true; + } + + @Override + void saveData(CollectRep.MetricsData metricsData) { + if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) { + return; + } + if (metricsData.getValuesList().isEmpty()) { + log.info("[warehouse influxdb] flush metrics data {} is null, ignore.", metricsData.getId()); + return; + } + List fieldsList = metricsData.getFieldsList(); + + String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getId()); + + List points = new ArrayList<>(); + for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + Point.Builder builder = Point.measurement(table); + builder.time(metricsData.getTime(), TimeUnit.MILLISECONDS); + String instance = valueRow.getInstance(); + builder.tag("instance", instance); + for (int i = 0; i < fieldsList.size(); i++) { + if (!CommonConstants.NULL_VALUE.equals(valueRow.getColumns(i))) { + if (fieldsList.get(i).getType() == CommonConstants.TYPE_NUMBER) { + builder.addField(fieldsList.get(i).getName(), Double.parseDouble(valueRow.getColumns(i))); + } else if (fieldsList.get(i).getType() == CommonConstants.TYPE_STRING) { + builder.addField(fieldsList.get(i).getName(), valueRow.getColumns(i)); + } + } else { + builder.addField(fieldsList.get(i).getName(), ""); + } + } + points.add(builder.build()); + } + BatchPoints.Builder builder = BatchPoints.database(DATABASE); + builder.points(points); + this.influxDb.write(builder.build()); + } + + @Override + public Map> getHistoryMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { + String table = this.generateTable(app, metrics, monitorId); + String selectSql = instance == null ? String.format(QUERY_HISTORY_SQL, metric, table, history) + : String.format(QUERY_HISTORY_SQL_WITH_INSTANCE, metric, table, instance, history); + Map> instanceValueMap = new HashMap<>(8); + try { + QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS); + for (QueryResult.Result result : selectResult.getResults()) { + if (result.getSeries() == null) { + continue; + } + for (QueryResult.Series series : result.getSeries()) { + for (List value : series.getValues()) { + long time = this.parseTimeToMillis(value.get(0)); + String instanceValue = value.get(1) == null ? "" : String.valueOf(value.get(1)); + String strValue = value.get(2) == null ? null : this.parseDoubleValue(value.get(2).toString()); + if (strValue == null) { + continue; + } + List valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); + valueList.add(new Value(strValue, time)); + } + } + } + } catch (Exception e) { + log.error("select history metric data in influxdb error, sql:{}, msg: {}", selectSql, e.getMessage()); + } + return instanceValueMap; + } + + @Override + public Map> getHistoryIntervalMetricData(Long monitorId, String app, String metrics, String metric, String instance, String history) { + String table = this.generateTable(app, metrics, monitorId); + Map> instanceValueMap = new HashMap<>(8); + Set instances = new HashSet<>(8); + if (instance != null) { + instances.add(instance); + } + if (instances.isEmpty()) { + // query the instance near 1week + String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table); + QueryResult instanceQueryResult = this.influxDb.query(new Query(queryInstanceSql, DATABASE), TimeUnit.MILLISECONDS); + for (QueryResult.Result result : instanceQueryResult.getResults()) { + if (result.getSeries() == null) { + continue; + } + for (QueryResult.Series series : result.getSeries()) { + for (List value : series.getValues()) { + if (value != null && value.get(1) != null) { + instances.add(value.get(1).toString()); + } + } + } + } + } + + try { + history = history.toLowerCase(); + if (instances.isEmpty()) { + instances.add(""); + } + for (String instanceValue : instances) { + String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, metric, metric, metric, table, instanceValue, history); + QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS); + for (QueryResult.Result result : selectResult.getResults()) { + if (result.getSeries() == null) { + continue; + } + for (QueryResult.Series series : result.getSeries()) { + for (List value : series.getValues()) { + Value.ValueBuilder valueBuilder = Value.builder(); + long time = this.parseTimeToMillis(value.get(0)); + valueBuilder.time(time); + + if (value.get(1) != null) { + valueBuilder.origin(this.parseDoubleValue(value.get(1).toString())); + } else { + continue; + } + if (value.get(2) != null) { + valueBuilder.mean(this.parseDoubleValue(value.get(2).toString())); + } else { + continue; + } + if (value.get(3) != null) { + valueBuilder.min(this.parseDoubleValue(value.get(3).toString())); + } else { + continue; + } + if (value.get(4) != null) { + valueBuilder.max(this.parseDoubleValue(value.get(4).toString())); + } else { + continue; + } + List valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); + valueList.add(valueBuilder.build()); + } + } + } + List instanceValueList = instanceValueMap.get(instanceValue); + if (instanceValueList == null || instanceValueList.isEmpty()) { + instanceValueMap.remove(instanceValue); + } + } + } catch (Exception e) { + log.error("select history interval metric data in influxdb error, msg: {}", e.getMessage()); + } + return instanceValueMap; + } + + private String generateTable(String app, String metrics, Long monitorId) { + return app + "_" + metrics + "_" + monitorId; + } + + private long parseTimeToMillis(Object time) { + if (time == null) { + return 0; + } + Double doubleTime = (Double) time; + return doubleTime.longValue(); + } + + private String parseDoubleValue(String value) { + return (new BigDecimal(value)).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + } + + private static X509TrustManager defaultTrustManager() { + return new X509TrustManager() { + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } + }; + } + + private static SSLSocketFactory defaultSslSocketFactory() { + try { + SSLContext sslContext = SSLContexts.createDefault(); + sslContext.init(null, new TrustManager[]{ + defaultTrustManager() + }, new SecureRandom()); + return sslContext.getSocketFactory(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static HostnameVerifier noopHostnameVerifier() { + return (s, sslSession) -> true; + } + + @Override + public void destroy() throws Exception { + if (this.influxDb != null) { + this.influxDb.close(); + } + } +} diff --git a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryTdEngineDataStorage.java b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryTdEngineDataStorage.java index 76c735ab82..36c1305a9e 100644 --- a/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryTdEngineDataStorage.java +++ b/warehouse/src/main/java/org/dromara/hertzbeat/warehouse/store/HistoryTdEngineDataStorage.java @@ -265,7 +265,7 @@ public class HistoryTdEngineDataStorage extends AbstractHistoryDataStorage { double value = resultSet.getDouble(3); String strValue = new BigDecimal(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); List valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>()); - valueList.add(new Value(strValue, ts.getTime())); + valueList.add(new Value(strValue, ts.getTime() / 100 * 100)); } resultSet.close(); return instanceValuesMap; @@ -355,7 +355,7 @@ public class HistoryTdEngineDataStorage extends AbstractHistoryDataStorage { Value value = Value.builder() .origin(originStr).mean(avgStr) .min(minStr).max(maxStr) - .time(ts.getTime()) + .time(ts.getTime() / 100 * 100) .build(); values.add(value); } diff --git a/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts b/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts index f943359e57..230bd3c2ff 100644 --- a/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts +++ b/web-app/src/app/routes/monitor/monitor-data-chart/monitor-data-chart.component.ts @@ -132,7 +132,7 @@ export class MonitorDataChartComponent implements OnInit { } }, onclick: () => { - this.loadData('1M', true); + this.loadData('4W', true); } }, myPeriod3m: { @@ -145,7 +145,7 @@ export class MonitorDataChartComponent implements OnInit { } }, onclick: () => { - this.loadData('3M', true); + this.loadData('12W', true); } }, myRefresh: {