wyl/hertzbeat
1
0
Fork 0
mirror of https://github.com/dromara/hertzbeat.git synced 2026-09-17 09:40:58 +00:00
Code Issues Projects Releases Wiki Activity

[fix] strip NUL bytes before tdengine history insert (#4310)

Browse Source 
Co-authored-by: aias00 <liuhongyu@apache.org>
This commit is contained in:
NekoPunch
2026-08-17 13:46:55 +08:00
committed by GitHub
co-authored by aias00
parent 1573236b15
commit 33284bb955
2 changed files with 32 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java
+6 -4
View File
@@ -66,6 +66,7 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage {
private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://";
private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')");
private static final Pattern NUL_CHAR_PATTERN = Pattern.compile("\\u0000");
private static final String INSTANCE_NULL = "''";
private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s";
private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s";
@@ -346,12 +347,13 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage {
}
private String formatStringValue(String value) {
String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
// bugfix Argument list too long
if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) {
// snmp octet strings may carry NUL padding that breaks the insert sql
String formatValue = NUL_CHAR_PATTERN.matcher(value).replaceAll("");
// truncate the logical value before escaping so the cut cannot split an escape sequence
if (formatValue.length() > tableStrColumnDefineMaxLength) {
formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength);
}
return formatValue;
return SQL_SPECIAL_STRING_PATTERN.matcher(formatValue).replaceAll("\\\\$0");
}
@Override
hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java
+26 -3
View File
@@ -115,6 +115,25 @@ class TdEngineDataStorageTest {
assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"), "Should contain timestamp and value 68.7");
}
@Test
void testSaveDataStripsControlCharacters() throws Exception {
tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties);
setPrivateField(tdEngineDataStorage, "hikariDataSource", mockHikariDataSource);
setParentPrivateField(tdEngineDataStorage, "serverAvailable", true);
// snmp octet strings can carry NUL bytes (issue #1481); tabs are legitimate data and stay
CollectRep.MetricsData metricsData = generateMockedMetricsData("Loopback\tInterface 1\u0000");
tdEngineDataStorage.saveData(metricsData);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture());
String executedSql = sqlCaptor.getValue();
assertTrue(executedSql.contains("Loopback\tInterface 1'"), "NUL stripped, tab preserved");
assertTrue(executedSql.indexOf('\u0000') < 0, "no raw NUL byte may reach the sql text");
assertTrue(!executedSql.contains("\\u0000"), "no textual NUL escape may survive in the labels json");
}
@Test
void destroy() {
}
@@ -146,6 +165,10 @@ class TdEngineDataStorageTest {
}
public static CollectRep.MetricsData generateMockedMetricsData() {
return generateMockedMetricsData("test-%server-01");
}
public static CollectRep.MetricsData generateMockedMetricsData(String instanceValue) {
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
when(mockMetricsData.getId()).thenReturn(0L);
@@ -156,9 +179,9 @@ class TdEngineDataStorageTest {
when(mockMetricsData.getInstance()).thenReturn("test-%server-01");
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
List<String> columnValues = List.of("test-%server-01", "68.7");
List<String> columnValues = List.of(instanceValue, "68.7");
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
when(mockValueRow.getColumns(0)).thenReturn("test-%server-01");
when(mockValueRow.getColumns(0)).thenReturn(instanceValue);
when(mockValueRow.getColumns(1)).thenReturn("68.7");
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
@@ -177,7 +200,7 @@ class TdEngineDataStorageTest {
Field instanceArrowField = new Field("instance", instanceFieldType, null);
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
when(instanceCell.getField()).thenReturn(instanceArrowField);
when(instanceCell.getValue()).thenReturn("test-%server-01");
when(instanceCell.getValue()).thenReturn(instanceValue);
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);