maintenance: classify service credentials as passwords (#4281)

Co-authored-by: Duansg <siguoduan@gmail.com>
This commit is contained in:
Logic
2026-08-27 09:50:46 +08:00
committed by GitHub
co-authored by Duansg
parent eb753be1f5
commit 1f257a8870
18 changed files with 838 additions and 13 deletions
@@ -37,7 +37,10 @@ public class PasswordParamValidator implements ParamValidator {
@Override
public void validate(ParamDefineInfo paramDefine, MonitorParam param) {
String passwordValue = param.getParamValue();
if (!AesUtil.isCiphertext(passwordValue)) {
boolean currentCiphertext = AesUtil.isCiphertext(passwordValue);
boolean legacyCiphertext = !AesUtil.DEFAULT_ENCODE_RULES.equals(AesUtil.getDefaultSecretKey())
&& AesUtil.isCiphertext(passwordValue, AesUtil.DEFAULT_ENCODE_RULES);
if (!currentCiphertext && !legacyCiphertext) {
passwordValue = AesUtil.aesEncode(passwordValue);
param.setParamValue(passwordValue);
}
@@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.util.AesUtil;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
/**
* Encrypts credentials that were stored before their template parameter type
* was classified as a password.
*/
@Component
@Order(Ordered.HIGHEST_PRECEDENCE)
@Slf4j
public class ServiceCredentialMigration implements CommandLineRunner {
private static final String MIGRATION_MARKER = "migration.service-credentials.v1";
private static final String FIND_CREDENTIALS_SQL = """
SELECT p.id, p.param_value, p.type
FROM hzb_param p
INNER JOIN hzb_monitor m ON m.id = p.monitor_id
WHERE (m.app = 'ollama' AND p.field = 'apiKey')
OR (m.scrape = 'http_sd' AND p.field = '__sd_token__')
""";
private static final String UPDATE_CREDENTIAL_SQL =
"UPDATE hzb_param SET param_value = ?, type = ? WHERE id = ?";
private final JdbcTemplate jdbcTemplate;
public ServiceCredentialMigration(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
@Transactional(rollbackFor = Exception.class)
public void run(String... args) {
if (isMigrationComplete()) {
return;
}
int migrated = migrateStoredCredentials();
jdbcTemplate.update("INSERT INTO hzb_config(type, content) VALUES (?, ?)",
MIGRATION_MARKER, "complete");
if (migrated > 0) {
log.info("Migrated {} stored service credential parameters", migrated);
}
}
private boolean isMigrationComplete() {
Integer markerCount = jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM hzb_config WHERE type = ?",
Integer.class,
MIGRATION_MARKER);
return markerCount != null && markerCount > 0;
}
int migrateStoredCredentials() {
List<StoredCredential> credentials = jdbcTemplate.query(FIND_CREDENTIALS_SQL,
(resultSet, rowNumber) -> new StoredCredential(
resultSet.getLong("id"),
resultSet.getString("param_value"),
resultSet.getByte("type")));
int migrated = 0;
for (StoredCredential credential : credentials) {
String value = credential.value();
boolean passwordType = credential.type() == CommonConstants.PARAM_TYPE_PASSWORD;
boolean encrypted = !StringUtils.hasText(value) || isSupportedCiphertext(value);
if (passwordType && encrypted) {
continue;
}
String valueToStore = value;
if (StringUtils.hasText(value) && !encrypted) {
valueToStore = AesUtil.aesEncode(value);
if (!AesUtil.isCiphertext(valueToStore)) {
throw new IllegalStateException("Could not encrypt a stored service credential");
}
}
jdbcTemplate.update(UPDATE_CREDENTIAL_SQL,
valueToStore,
CommonConstants.PARAM_TYPE_PASSWORD,
credential.id());
migrated++;
}
return migrated;
}
private boolean isSupportedCiphertext(String value) {
if (AesUtil.isCiphertext(value)) {
return true;
}
return !AesUtil.DEFAULT_ENCODE_RULES.equals(AesUtil.getDefaultSecretKey())
&& AesUtil.isCiphertext(value, AesUtil.DEFAULT_ENCODE_RULES);
}
private record StoredCredential(long id, String value, byte type) {
}
}
@@ -91,9 +91,13 @@ public class MonitorDto {
}
public void setParams(List<Param> params) {
setParams(params, true);
}
public void setParams(List<Param> params, boolean maskCredentials) {
this.paramInfos = params == null ? null : params.stream()
.filter(Objects::nonNull)
.map(MonitorParam::fromEntity)
.map(param -> MonitorParam.fromEntity(param, maskCredentials))
.toList();
}
@@ -24,6 +24,7 @@ import java.time.LocalDateTime;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.manager.Param;
/**
@@ -34,6 +35,8 @@ import org.apache.hertzbeat.common.entity.manager.Param;
@NoArgsConstructor
public class MonitorParam {
public static final String SECRET_MASK = "******";
private Long id;
private Long monitorId;
@@ -53,6 +56,10 @@ public class MonitorParam {
private LocalDateTime gmtUpdate;
public static MonitorParam fromEntity(Param param) {
return fromEntity(param, true);
}
public static MonitorParam fromEntity(Param param, boolean maskCredential) {
if (param == null) {
return null;
}
@@ -60,13 +67,24 @@ public class MonitorParam {
monitorParam.setId(param.getId());
monitorParam.setMonitorId(param.getMonitorId());
monitorParam.setField(param.getField());
monitorParam.setParamValue(param.getParamValue());
String value = param.getParamValue();
if (maskCredential
&& param.getType() == CommonConstants.PARAM_TYPE_PASSWORD
&& value != null
&& !value.isEmpty()) {
value = SECRET_MASK;
}
monitorParam.setParamValue(value);
monitorParam.setType(param.getType());
monitorParam.setGmtCreate(param.getGmtCreate());
monitorParam.setGmtUpdate(param.getGmtUpdate());
return monitorParam;
}
public static boolean isSecretMask(String value) {
return SECRET_MASK.equals(value);
}
public Param toEntity() {
Param param = new Param();
param.setId(id);
@@ -102,6 +102,16 @@ public interface MonitorService {
*/
MonitorDto getMonitorDto(long id) throws RuntimeException;
/**
* Get monitoring information for a portable configuration export. Password
* values remain encrypted so the exported configuration can be imported by
* an installation configured with the same common secret.
*
* @param id Monitor ID
* @return monitor information with encrypted parameter values
*/
MonitorDto getMonitorDtoForExport(long id) throws RuntimeException;
/**
* Dynamic conditional query
*
@@ -98,7 +98,11 @@ public abstract class AbstractImExportServiceImpl implements ImExportService {
@Override
public void exportConfig(OutputStream os, List<Long> configList) {
var monitorList = configList.stream().map(it -> monitorService.getMonitorDto(it)).filter(Objects::nonNull).map(this::convert).toList();
var monitorList = configList.stream()
.map(monitorService::getMonitorDtoForExport)
.filter(Objects::nonNull)
.map(this::convert)
.toList();
writeOs(monitorList, os);
}
@@ -168,7 +172,7 @@ public abstract class AbstractImExportServiceImpl implements ImExportService {
param.setType(it.type);
param.setParamValue(it.value);
return param;
}).toList());
}).toList(), false);
} else {
monitorDto.setParams(Collections.emptyList());
}
@@ -43,6 +43,7 @@ import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.support.event.MonitorDeletedEvent;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.common.util.JexlCheckerUtil;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
@@ -103,6 +104,10 @@ public class MonitorServiceImpl implements MonitorService {
public static final String PATTERN_HTTPS = "(?i)https://";
private static final Long MONITOR_ID_TMP = 1000000000L;
private static final byte ALL_MONITOR_STATUS = 9;
private static final Map<String, Set<String>> CREDENTIAL_DESTINATION_FIELDS = Map.of(
"apiKey", Set.of("host", "port", "ssl"),
"__sd_token__", Set.of("__sd_url__", "__sd_authType__"),
"__sd_password__", Set.of("__sd_url__", "__sd_authType__"));
public static final String PARAM_FIELD_PORT = "port";
@Autowired
@@ -323,10 +328,33 @@ public class MonitorServiceImpl implements MonitorService {
monitorDto.setCollector(null);
}
// Parameter definition structure verification
List<ParamDefineInfo> paramDefines = appService.getAppParamDefines(monitor.getApp());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
List<ParamDefineInfo> paramDefines = new ArrayList<>();
List<ParamDefineInfo> applicationParamDefines = appService.getAppParamDefines(monitor.getApp());
if (!CollectionUtils.isEmpty(applicationParamDefines)) {
paramDefines.addAll(applicationParamDefines);
}
if (!isStatic && !Objects.equals(monitor.getApp(), monitor.getScrape())) {
List<ParamDefineInfo> scrapeParamDefines = appService.getAppParamDefines(monitor.getScrape());
if (!CollectionUtils.isEmpty(scrapeParamDefines)) {
paramDefines.addAll(scrapeParamDefines);
}
}
boolean restoresMaskedCredential = (Boolean.TRUE.equals(isModify)
|| (isModify == null && monitor.getId() != null))
&& !CollectionUtils.isEmpty(paramDefines)
&& paramDefines.stream()
.filter(paramDefine -> "password".equals(paramDefine.getType()))
.map(ParamDefineInfo::getField)
.map(paramMap::get)
.filter(Objects::nonNull)
.anyMatch(param -> MonitorParam.isSecretMask(param.getParamValue()));
Map<String, Param> storedParams = restoresMaskedCredential
? paramDao.findParamsByMonitorId(monitor.getId()).stream()
.collect(Collectors.toMap(Param::getField, param -> param))
: Map.of();
if (!CollectionUtils.isEmpty(paramDefines)) {
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
for (ParamDefineInfo paramDefine : paramDefines) {
String field = paramDefine.getField();
MonitorParam param = paramMap.get(field);
@@ -334,6 +362,27 @@ public class MonitorServiceImpl implements MonitorService {
if (!isStatic && "host".equals(field)) {
continue;
}
if ("password".equals(paramDefine.getType())
&& param != null
&& MonitorParam.isSecretMask(param.getParamValue())) {
if (!restoresMaskedCredential) {
throw new IllegalArgumentException("The credential mask cannot be used as a new value.");
}
Param storedParam = storedParams.get(field);
if (storedParam == null || !StringUtils.hasText(storedParam.getParamValue())) {
throw new IllegalArgumentException("The masked credential has no stored value.");
}
String storedValue = storedParam.getParamValue();
boolean currentCiphertext = AesUtil.isCiphertext(storedValue);
boolean legacyCiphertext = !AesUtil.DEFAULT_ENCODE_RULES.equals(AesUtil.getDefaultSecretKey())
&& AesUtil.isCiphertext(storedValue, AesUtil.DEFAULT_ENCODE_RULES);
if (!currentCiphertext && !legacyCiphertext) {
throw new IllegalStateException("The stored credential migration is incomplete.");
}
validateCredentialDestination(field, paramMap, storedParams);
param.setParamValue(storedValue);
param.setType(CommonConstants.PARAM_TYPE_PASSWORD);
}
if (paramDefine.isRequired() && (param == null || param.getParamValue() == null)) {
throw new IllegalArgumentException("Params field " + field + " is required.");
}
@@ -342,7 +391,27 @@ public class MonitorServiceImpl implements MonitorService {
}
}
}
checkJobFields(monitorDto.getMonitor().getApp());
checkJobFields(monitor.getApp());
if (!isStatic && !Objects.equals(monitor.getApp(), monitor.getScrape())) {
checkJobFields(monitor.getScrape());
}
}
private void validateCredentialDestination(
String credentialField,
Map<String, MonitorParam> submittedParams,
Map<String, Param> storedParams) {
for (String destinationField : CREDENTIAL_DESTINATION_FIELDS.getOrDefault(
credentialField, Set.of())) {
MonitorParam submitted = submittedParams.get(destinationField);
Param stored = storedParams.get(destinationField);
String submittedValue = submitted == null ? null : submitted.getParamValue();
String storedValue = stored == null ? null : stored.getParamValue();
if (!Objects.equals(submittedValue, storedValue)) {
throw new IllegalArgumentException(
"A credential must be re-entered when its destination changes.");
}
}
}
private void checkJobFields(String app) {
@@ -509,6 +578,16 @@ public class MonitorServiceImpl implements MonitorService {
@Override
@Transactional(readOnly = true)
public MonitorDto getMonitorDto(long id) throws RuntimeException {
return getMonitorDto(id, true);
}
@Override
@Transactional(readOnly = true)
public MonitorDto getMonitorDtoForExport(long id) throws RuntimeException {
return getMonitorDto(id, false);
}
private MonitorDto getMonitorDto(long id, boolean maskCredentials) throws RuntimeException {
Optional<Monitor> monitorOptional = monitorDao.findById(id);
if (monitorOptional.isPresent()) {
// Get current user ID for favorite status
@@ -524,7 +603,7 @@ public class MonitorServiceImpl implements MonitorService {
Monitor monitor = monitorOptional.get();
MonitorDto monitorDto = new MonitorDto();
List<Param> params = paramDao.findParamsByMonitorId(id);
monitorDto.setParams(params);
monitorDto.setParams(params, maskCredentials);
List<MetricsInfo> metricsInfos;
if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(monitor.getApp())
|| monitor.getType() == CommonConstants.MONITOR_TYPE_PUSH_AUTO_CREATE) {
@@ -62,7 +62,7 @@ params:
en-US: Access Token
ja-JP: アクセストークン
# type-param field type(most mapping the html input type)
type: text
type: password
# dependent parameter values list
depend:
__sd_authType__:
@@ -65,7 +65,7 @@ params:
zh-CN: API Key
en-US: API Key
ja-JP: API キー
type: text
type: password
required: false
# collect metrics config list
metrics:
@@ -72,4 +72,24 @@ class PasswordParamValidatorTest {
assertEquals(ciphertext, param.getParamValue());
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, param.getType());
}
@Test
void validate_IgnoresLegacyCiphertextUntilMigrationReencryptsIt() {
String originalSecret = AesUtil.getDefaultSecretKey();
try {
AesUtil.setDefaultSecretKey("nextSecretKey123");
String legacyCiphertext = AesUtil.aesEncode("password123", AesUtil.DEFAULT_ENCODE_RULES);
ParamDefineInfo paramDefine = new ParamDefineInfo();
paramDefine.setType("password");
MonitorParam param = new MonitorParam();
param.setParamValue(legacyCiphertext);
validator.validate(paramDefine, param);
assertEquals(legacyCiphertext, param.getParamValue());
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, param.getType());
} finally {
AesUtil.setDefaultSecretKey(originalSecret);
}
}
}
@@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Map;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.util.AesUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
class ServiceCredentialMigrationTest {
private static final String TEST_SECRET = "0123456789abcdef";
private JdbcTemplate jdbcTemplate;
@BeforeEach
void setUp() {
AesUtil.setDefaultSecretKey(TEST_SECRET);
DriverManagerDataSource dataSource = new DriverManagerDataSource(
"jdbc:h2:mem:credential-migration;MODE=MySQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE",
"sa",
"");
jdbcTemplate = new JdbcTemplate(dataSource);
jdbcTemplate.execute("DROP ALL OBJECTS");
jdbcTemplate.execute("""
CREATE TABLE hzb_monitor (
id BIGINT PRIMARY KEY,
app VARCHAR(100),
scrape VARCHAR(100)
)
""");
jdbcTemplate.execute("""
CREATE TABLE hzb_param (
id BIGINT PRIMARY KEY,
monitor_id BIGINT,
field VARCHAR(100),
param_value VARCHAR(8126),
type TINYINT
)
""");
jdbcTemplate.execute("""
CREATE TABLE hzb_config (
type VARCHAR(100) PRIMARY KEY,
content VARCHAR(8192)
)
""");
}
@AfterEach
void tearDown() {
AesUtil.setDefaultSecretKey(AesUtil.DEFAULT_ENCODE_RULES);
}
@Test
void migratesLegacyCredentialsInTheDatabaseIdempotently() {
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 1L, "ollama", "static");
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 2L, "linux", "http_sd");
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 3L, "website", "static");
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 4L, "ollama", "static");
String alreadyEncrypted = AesUtil.aesEncode("existing-ciphertext");
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
11L, 1L, "apiKey", "legacy-ollama-key", CommonConstants.PARAM_TYPE_STRING);
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
12L, 2L, "__sd_token__", "legacy-http-sd-token", CommonConstants.PARAM_TYPE_STRING);
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
13L, 3L, "apiKey", "ordinary-text", CommonConstants.PARAM_TYPE_STRING);
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
14L, 4L, "apiKey", alreadyEncrypted, CommonConstants.PARAM_TYPE_PASSWORD);
ServiceCredentialMigration migration = new ServiceCredentialMigration(jdbcTemplate);
assertEquals(2, migration.migrateStoredCredentials());
Map<String, Object> ollama = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 11");
Map<String, Object> httpSd = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 12");
Map<String, Object> ordinary = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 13");
Map<String, Object> encrypted = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 14");
String ollamaCiphertext = String.valueOf(ollama.get("PARAM_VALUE"));
String httpSdCiphertext = String.valueOf(httpSd.get("PARAM_VALUE"));
assertNotEquals("legacy-ollama-key", ollamaCiphertext);
assertNotEquals("legacy-http-sd-token", httpSdCiphertext);
assertTrue(AesUtil.isCiphertext(ollamaCiphertext));
assertTrue(AesUtil.isCiphertext(httpSdCiphertext));
assertEquals("legacy-ollama-key", AesUtil.aesDecode(ollamaCiphertext));
assertEquals("legacy-http-sd-token", AesUtil.aesDecode(httpSdCiphertext));
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, ((Number) ollama.get("TYPE")).byteValue());
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, ((Number) httpSd.get("TYPE")).byteValue());
assertEquals("ordinary-text", ordinary.get("PARAM_VALUE"));
assertEquals(CommonConstants.PARAM_TYPE_STRING, ((Number) ordinary.get("TYPE")).byteValue());
assertEquals(alreadyEncrypted, encrypted.get("PARAM_VALUE"));
assertEquals(0, migration.migrateStoredCredentials());
assertEquals(ollamaCiphertext, jdbcTemplate.queryForObject(
"SELECT param_value FROM hzb_param WHERE id = 11", String.class));
}
@Test
void startupMigrationRunsOnlyOnceAndDoesNotRewriteLaterCredentials() {
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)",
1L, "ollama", "static");
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
11L, 1L, "apiKey", "legacy-ollama-key", CommonConstants.PARAM_TYPE_STRING);
ServiceCredentialMigration migration = new ServiceCredentialMigration(jdbcTemplate);
migration.run();
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
12L, 1L, "apiKey", "later-operator-value", CommonConstants.PARAM_TYPE_STRING);
migration.run();
assertTrue(AesUtil.isCiphertext(jdbcTemplate.queryForObject(
"SELECT param_value FROM hzb_param WHERE id = 11", String.class)));
assertEquals("later-operator-value", jdbcTemplate.queryForObject(
"SELECT param_value FROM hzb_param WHERE id = 12", String.class));
assertEquals(1, jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM hzb_config WHERE type = 'migration.service-credentials.v1'",
Integer.class));
}
}
@@ -123,13 +123,18 @@ class MonitorControllerTest {
MonitorDto monitorDto = new MonitorDto();
monitorDto.setMonitor(monitor);
Param secret = new Param();
secret.setField("apiKey");
secret.setParamValue("ciphertext-must-not-leave-the-api");
secret.setType(CommonConstants.PARAM_TYPE_PASSWORD);
monitorDto.setParams(List.of(secret));
Mockito.when(monitorService.getMonitorDto(6565463543L))
.thenReturn(monitorDto);
this.mockMvc.perform(MockMvcRequestBuilders.get("/api/monitor/{id}", 6565463543L))
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.data.params[0].paramValue").value("******"))
.andExpect(status().isOk())
.andReturn();
}
@@ -46,6 +46,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -85,6 +86,12 @@ class AppServiceTest {
assertDoesNotThrow(() -> appService.getAppParamDefines("jvm"));
}
@Test
void credentialParametersUsePasswordType() {
assertEquals("password", findParam("ollama", "apiKey").getType());
assertEquals("password", findParam("http_sd", "__sd_token__").getType());
}
@Test
void getAppParamDefinesShouldNotContainDuplicateFields() {
List<ParamDefineInfo> paramDefines = appService.getAppParamDefines("nvidia");
@@ -203,4 +210,11 @@ class AppServiceTest {
assertNotNull(appParamDefines);
assertTrue(appParamDefines.stream().anyMatch(t -> t.getField().equals("host_test")));
}
private ParamDefineInfo findParam(String app, String field) {
return appService.getAppParamDefines(app).stream()
.filter(param -> field.equals(param.getField()))
.findFirst()
.orElseThrow();
}
}
@@ -32,6 +32,8 @@ import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.manager.config.ManagerSseManager;
import org.apache.hertzbeat.manager.service.impl.AbstractImExportServiceImpl;
import org.apache.hertzbeat.manager.service.impl.JsonImExportServiceImpl;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.pojo.dto.MonitorParam;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -110,6 +112,41 @@ class JsonImExportServiceTest {
assertEquals("JSON", jsonImExportService.type());
}
@Test
void testExportConfigPreservesEncryptedCredentialForImportRoundTrip() {
String ciphertext = "HBA2-export-ciphertext";
MonitorDto monitorDto = new MonitorDto();
monitorDto.setMonitor(Monitor.builder().id(1L).name("ollama").app("ollama").build());
MonitorParam secret = new MonitorParam();
secret.setField("apiKey");
secret.setType(org.apache.hertzbeat.common.constants.CommonConstants.PARAM_TYPE_PASSWORD);
secret.setParamValue(ciphertext);
monitorDto.setParamInfos(List.of(secret));
org.mockito.Mockito.when(monitorService.getMonitorDtoForExport(1L)).thenReturn(monitorDto);
ByteArrayOutputStream output = new ByteArrayOutputStream();
jsonImExportService.exportConfig(output, List.of(1L));
assertTrue(output.toString(StandardCharsets.UTF_8).contains(ciphertext));
}
@Test
void testImportConfigPreservesEncryptedCredentialForImportRoundTrip() {
String ciphertext = "HBA2-import-ciphertext";
String json = "[{\"monitor\":{\"name\":\"ollama-import\",\"app\":\"ollama\","
+ "\"intervals\":6000,\"status\":1},\"params\":[{\"field\":\"apiKey\","
+ "\"type\":2,\"value\":\"" + ciphertext + "\"}]}]";
ArgumentCaptor<List<Param>> paramsCaptor = ArgumentCaptor.forClass(List.class);
doNothing().when(monitorService).addMonitor(
org.mockito.Mockito.any(), paramsCaptor.capture(),
org.mockito.Mockito.any(), org.mockito.Mockito.any());
jsonImExportService.importConfig(
"ollama.json", new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)));
assertEquals(ciphertext, paramsCaptor.getValue().get(0).getParamValue());
}
@Test
void testImportConfig_shouldSetInstanceFromHostAndPortParams() {
String json = "[{\"monitor\":{\"name\":\"test\",\"app\":\"windows\",\"intervals\":6000,\"status\":1},"
@@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
@@ -42,6 +44,7 @@ import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorBindDao;
@@ -49,6 +52,7 @@ import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.pojo.dto.AppCount;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.pojo.dto.MonitorParam;
import org.apache.hertzbeat.manager.pojo.dto.ParamDefineInfo;
import org.apache.hertzbeat.manager.scheduler.CollectJobScheduling;
import org.apache.hertzbeat.manager.component.validator.ParamValidatorManager;
@@ -161,6 +165,242 @@ class MonitorServiceTest {
return paramDefine;
}
@Test
void validateModifyRestoresTheStoredCredentialBehindTheResponseMask() {
long monitorId = 101L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("ollama-local")
.app("ollama")
.scrape("static")
.instance("127.0.0.1")
.intervals(60)
.build();
Param submitted = Param.builder()
.monitorId(monitorId)
.field("apiKey")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
String storedCiphertext = AesUtil.aesEncode("stored-ollama-key");
Param stored = Param.builder()
.id(11L)
.monitorId(monitorId)
.field("apiKey")
.paramValue(storedCiphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submitted));
when(appService.getAppParamDefines("ollama"))
.thenReturn(List.of(newParamDefine("apiKey", "password", false)));
when(appService.getAppDefineOption("ollama-local")).thenReturn(Optional.empty());
when(monitorDao.findMonitorByNameEquals("ollama-local")).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(stored));
Job job = new Job();
job.setMetrics(Collections.emptyList());
when(appService.getAppDefine("ollama")).thenReturn(job);
monitorService.validate(dto, true);
ArgumentCaptor<MonitorParam> paramCaptor = ArgumentCaptor.forClass(MonitorParam.class);
verify(paramValidatorManager).validate(any(ParamDefineInfo.class), paramCaptor.capture());
assertEquals(storedCiphertext, paramCaptor.getValue().getParamValue());
assertEquals(storedCiphertext, dto.getParams().get(0).getParamValue());
}
@Test
void validateDetectRestoresStoredCredentialForAnExistingMonitor() {
long monitorId = 103L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("ollama-detect")
.app("ollama")
.scrape("static")
.instance("127.0.0.1")
.intervals(60)
.build();
Param submitted = Param.builder()
.monitorId(monitorId)
.field("apiKey")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
String storedCiphertext = AesUtil.aesEncode("stored-detect-key");
Param stored = Param.builder()
.id(31L)
.monitorId(monitorId)
.field("apiKey")
.paramValue(storedCiphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submitted));
when(appService.getAppParamDefines("ollama"))
.thenReturn(List.of(newParamDefine("apiKey", "password", false)));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(stored));
Job job = new Job();
job.setMetrics(Collections.emptyList());
when(appService.getAppDefine("ollama")).thenReturn(job);
monitorService.validate(dto, null);
assertEquals(storedCiphertext, dto.getParams().get(0).getParamValue());
verify(paramValidatorManager).validate(any(ParamDefineInfo.class), any(MonitorParam.class));
}
@Test
void validateModifyRejectsMaskedCredentialWhenDestinationChanges() {
long monitorId = 102L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("ollama-remote")
.app("ollama")
.scrape("static")
.instance("attacker.example")
.intervals(60)
.build();
Param submittedHost = Param.builder()
.monitorId(monitorId)
.field("host")
.paramValue("attacker.example")
.type(CommonConstants.PARAM_TYPE_STRING)
.build();
Param submittedSecret = Param.builder()
.monitorId(monitorId)
.field("apiKey")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
Param storedHost = Param.builder()
.id(21L)
.monitorId(monitorId)
.field("host")
.paramValue("trusted.example")
.type(CommonConstants.PARAM_TYPE_STRING)
.build();
Param storedSecret = Param.builder()
.id(22L)
.monitorId(monitorId)
.field("apiKey")
.paramValue(AesUtil.aesEncode("stored-ollama-key"))
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submittedHost, submittedSecret));
when(appService.getAppParamDefines("ollama")).thenReturn(List.of(
newParamDefine("host", "host", true),
newParamDefine("apiKey", "password", false)));
when(appService.getAppDefineOption("ollama-remote")).thenReturn(Optional.empty());
when(monitorDao.findMonitorByNameEquals("ollama-remote")).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(storedHost, storedSecret));
assertThrows(IllegalArgumentException.class, () -> monitorService.validate(dto, true));
}
@Test
void validateModifyRestoresHttpServiceDiscoveryCredentialFromScrapeDefinition() {
long monitorId = 103L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("discovered-prometheus")
.app("prometheus")
.scrape("http_sd")
.instance("https://discovery.example")
.intervals(60)
.build();
Param submittedUrl = Param.builder()
.monitorId(monitorId)
.field("__sd_url__")
.paramValue("https://discovery.example")
.type(CommonConstants.PARAM_TYPE_STRING)
.build();
Param submittedToken = Param.builder()
.monitorId(monitorId)
.field("__sd_token__")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
String storedCiphertext = AesUtil.aesEncode("stored-discovery-token");
Param storedUrl = submittedUrl.clone();
Param storedToken = Param.builder()
.id(32L)
.monitorId(monitorId)
.field("__sd_token__")
.paramValue(storedCiphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submittedUrl, submittedToken));
when(appService.getAppParamDefines("prometheus")).thenReturn(Collections.emptyList());
when(appService.getAppParamDefines("http_sd")).thenReturn(List.of(
newParamDefine("__sd_url__", "text", true),
newParamDefine("__sd_token__", "password", false)));
when(appService.getAppDefineOption("discovered-prometheus")).thenReturn(Optional.empty());
when(monitorDao.findMonitorByNameEquals("discovered-prometheus")).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(storedUrl, storedToken));
Job applicationJob = new Job();
applicationJob.setMetrics(Collections.emptyList());
when(appService.getAppDefine("prometheus")).thenReturn(applicationJob);
Job job = new Job();
job.setMetrics(Collections.emptyList());
when(appService.getAppDefine("http_sd")).thenReturn(job);
monitorService.validate(dto, true);
assertEquals(storedCiphertext, dto.getParams().stream()
.filter(param -> "__sd_token__".equals(param.getField()))
.findFirst()
.orElseThrow()
.getParamValue());
}
@Test
void validateServiceDiscoveryMonitorChecksApplicationAndScrapeCredentials() {
Monitor monitor = Monitor.builder()
.id(104L)
.name("discovered-mysql")
.app("mysql")
.scrape("http_sd")
.instance("https://discovery.example")
.intervals(60)
.build();
Param submittedPassword = Param.builder()
.monitorId(monitor.getId())
.field("password")
.paramValue("database-secret")
.build();
Param submittedDiscoveryToken = Param.builder()
.monitorId(monitor.getId())
.field("__sd_token__")
.paramValue("discovery-secret")
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submittedPassword, submittedDiscoveryToken));
ParamDefineInfo applicationPassword = newParamDefine("password", "password", true);
ParamDefineInfo discoveryToken = newParamDefine("__sd_token__", "password", true);
when(appService.getAppParamDefines("mysql")).thenReturn(List.of(applicationPassword));
when(appService.getAppParamDefines("http_sd")).thenReturn(List.of(discoveryToken));
Job applicationJob = new Job();
applicationJob.setMetrics(Collections.emptyList());
Job scrapeJob = new Job();
scrapeJob.setMetrics(Collections.emptyList());
when(appService.getAppDefine("mysql")).thenReturn(applicationJob);
when(appService.getAppDefine("http_sd")).thenReturn(scrapeJob);
monitorService.validate(dto, null);
verify(paramValidatorManager).validate(
eq(applicationPassword), argThat(param -> "password".equals(param.getField())));
verify(paramValidatorManager).validate(
eq(discoveryToken), argThat(param -> "__sd_token__".equals(param.getField())));
}
@Test
void detectMonitorEmpty() {
Monitor monitor = Monitor.builder()
@@ -775,6 +1015,38 @@ class MonitorServiceTest {
assertNotNull(monitorDto);
}
@Test
void getMonitorDtoMasksCredentialsButExportKeepsCiphertext() {
long id = 2L;
Monitor monitor = Monitor.builder()
.jobId(id)
.intervals(1)
.app("ollama")
.name("ollama-export")
.instance("localhost")
.id(id)
.build();
String ciphertext = AesUtil.aesEncode("portable-secret");
Param secret = Param.builder()
.monitorId(id)
.field("apiKey")
.paramValue(ciphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
when(monitorDao.findById(id)).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(id)).thenReturn(List.of(secret));
Job job = new Job();
job.setMetrics(new ArrayList<>());
when(appService.getAppDefine(monitor.getApp())).thenReturn(job);
when(collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(id)).thenReturn(Optional.empty());
MonitorDto response = monitorService.getMonitorDto(id);
MonitorDto export = monitorService.getMonitorDtoForExport(id);
assertEquals(MonitorParam.SECRET_MASK, response.getParamInfos().getFirst().getParamValue());
assertEquals(ciphertext, export.getParamInfos().getFirst().getParamValue());
}
@Test
void getMonitors() {
when(monitorDao.findAll(any(Specification.class), any(PageRequest.class))).thenAnswer((invocation) -> {