maintenance: classify service credentials as passwords (#4281)

Co-authored-by: Duansg <siguoduan@gmail.com>
This commit is contained in:
Logic
2026-08-27 09:50:46 +08:00
committed by GitHub
co-authored by Duansg
parent eb753be1f5
commit 1f257a8870
18 changed files with 838 additions and 13 deletions
@@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.timer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.List;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Configmap;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
import org.apache.hertzbeat.common.util.AesUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
class WheelTimerTaskCredentialTest {
private static final String TEST_SECRET = "0123456789abcdef";
@AfterEach
void tearDown() {
AesUtil.setDefaultSecretKey(AesUtil.DEFAULT_ENCODE_RULES);
}
@Test
void decryptsTheMigratedCredentialBeforeProtocolReplacement() {
AesUtil.setDefaultSecretKey(TEST_SECRET);
String ciphertext = AesUtil.aesEncode("runtime-ollama-key");
HttpProtocol.Authorization authorization = new HttpProtocol.Authorization();
authorization.setType("Bearer Token");
authorization.setBearerTokenToken("^_^apiKey^_^");
Metrics metrics = Metrics.builder()
.name("version")
.interval(60)
.http(HttpProtocol.builder().authorization(authorization).build())
.build();
Job job = Job.builder()
.app("ollama")
.defaultInterval(60)
.configmap(List.of(new Configmap(
"apiKey",
ciphertext,
CommonConstants.PARAM_TYPE_PASSWORD)))
.metrics(List.of(metrics))
.build();
new WheelTimerTask(job, timeout -> {
});
assertEquals("runtime-ollama-key",
job.getMetrics().get(0).getHttp().getAuthorization().getBearerTokenToken());
}
}
@@ -37,7 +37,10 @@ public class PasswordParamValidator implements ParamValidator {
@Override
public void validate(ParamDefineInfo paramDefine, MonitorParam param) {
String passwordValue = param.getParamValue();
if (!AesUtil.isCiphertext(passwordValue)) {
boolean currentCiphertext = AesUtil.isCiphertext(passwordValue);
boolean legacyCiphertext = !AesUtil.DEFAULT_ENCODE_RULES.equals(AesUtil.getDefaultSecretKey())
&& AesUtil.isCiphertext(passwordValue, AesUtil.DEFAULT_ENCODE_RULES);
if (!currentCiphertext && !legacyCiphertext) {
passwordValue = AesUtil.aesEncode(passwordValue);
param.setParamValue(passwordValue);
}
@@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.util.AesUtil;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;
/**
* Encrypts credentials that were stored before their template parameter type
* was classified as a password.
*/
@Component
@Order(Ordered.HIGHEST_PRECEDENCE)
@Slf4j
public class ServiceCredentialMigration implements CommandLineRunner {
private static final String MIGRATION_MARKER = "migration.service-credentials.v1";
private static final String FIND_CREDENTIALS_SQL = """
SELECT p.id, p.param_value, p.type
FROM hzb_param p
INNER JOIN hzb_monitor m ON m.id = p.monitor_id
WHERE (m.app = 'ollama' AND p.field = 'apiKey')
OR (m.scrape = 'http_sd' AND p.field = '__sd_token__')
""";
private static final String UPDATE_CREDENTIAL_SQL =
"UPDATE hzb_param SET param_value = ?, type = ? WHERE id = ?";
private final JdbcTemplate jdbcTemplate;
public ServiceCredentialMigration(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
@Transactional(rollbackFor = Exception.class)
public void run(String... args) {
if (isMigrationComplete()) {
return;
}
int migrated = migrateStoredCredentials();
jdbcTemplate.update("INSERT INTO hzb_config(type, content) VALUES (?, ?)",
MIGRATION_MARKER, "complete");
if (migrated > 0) {
log.info("Migrated {} stored service credential parameters", migrated);
}
}
private boolean isMigrationComplete() {
Integer markerCount = jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM hzb_config WHERE type = ?",
Integer.class,
MIGRATION_MARKER);
return markerCount != null && markerCount > 0;
}
int migrateStoredCredentials() {
List<StoredCredential> credentials = jdbcTemplate.query(FIND_CREDENTIALS_SQL,
(resultSet, rowNumber) -> new StoredCredential(
resultSet.getLong("id"),
resultSet.getString("param_value"),
resultSet.getByte("type")));
int migrated = 0;
for (StoredCredential credential : credentials) {
String value = credential.value();
boolean passwordType = credential.type() == CommonConstants.PARAM_TYPE_PASSWORD;
boolean encrypted = !StringUtils.hasText(value) || isSupportedCiphertext(value);
if (passwordType && encrypted) {
continue;
}
String valueToStore = value;
if (StringUtils.hasText(value) && !encrypted) {
valueToStore = AesUtil.aesEncode(value);
if (!AesUtil.isCiphertext(valueToStore)) {
throw new IllegalStateException("Could not encrypt a stored service credential");
}
}
jdbcTemplate.update(UPDATE_CREDENTIAL_SQL,
valueToStore,
CommonConstants.PARAM_TYPE_PASSWORD,
credential.id());
migrated++;
}
return migrated;
}
private boolean isSupportedCiphertext(String value) {
if (AesUtil.isCiphertext(value)) {
return true;
}
return !AesUtil.DEFAULT_ENCODE_RULES.equals(AesUtil.getDefaultSecretKey())
&& AesUtil.isCiphertext(value, AesUtil.DEFAULT_ENCODE_RULES);
}
private record StoredCredential(long id, String value, byte type) {
}
}
@@ -91,9 +91,13 @@ public class MonitorDto {
}
public void setParams(List<Param> params) {
setParams(params, true);
}
public void setParams(List<Param> params, boolean maskCredentials) {
this.paramInfos = params == null ? null : params.stream()
.filter(Objects::nonNull)
.map(MonitorParam::fromEntity)
.map(param -> MonitorParam.fromEntity(param, maskCredentials))
.toList();
}
@@ -24,6 +24,7 @@ import java.time.LocalDateTime;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.manager.Param;
/**
@@ -34,6 +35,8 @@ import org.apache.hertzbeat.common.entity.manager.Param;
@NoArgsConstructor
public class MonitorParam {
public static final String SECRET_MASK = "******";
private Long id;
private Long monitorId;
@@ -53,6 +56,10 @@ public class MonitorParam {
private LocalDateTime gmtUpdate;
public static MonitorParam fromEntity(Param param) {
return fromEntity(param, true);
}
public static MonitorParam fromEntity(Param param, boolean maskCredential) {
if (param == null) {
return null;
}
@@ -60,13 +67,24 @@ public class MonitorParam {
monitorParam.setId(param.getId());
monitorParam.setMonitorId(param.getMonitorId());
monitorParam.setField(param.getField());
monitorParam.setParamValue(param.getParamValue());
String value = param.getParamValue();
if (maskCredential
&& param.getType() == CommonConstants.PARAM_TYPE_PASSWORD
&& value != null
&& !value.isEmpty()) {
value = SECRET_MASK;
}
monitorParam.setParamValue(value);
monitorParam.setType(param.getType());
monitorParam.setGmtCreate(param.getGmtCreate());
monitorParam.setGmtUpdate(param.getGmtUpdate());
return monitorParam;
}
public static boolean isSecretMask(String value) {
return SECRET_MASK.equals(value);
}
public Param toEntity() {
Param param = new Param();
param.setId(id);
@@ -102,6 +102,16 @@ public interface MonitorService {
*/
MonitorDto getMonitorDto(long id) throws RuntimeException;
/**
* Get monitoring information for a portable configuration export. Password
* values remain encrypted so the exported configuration can be imported by
* an installation configured with the same common secret.
*
* @param id Monitor ID
* @return monitor information with encrypted parameter values
*/
MonitorDto getMonitorDtoForExport(long id) throws RuntimeException;
/**
* Dynamic conditional query
*
@@ -98,7 +98,11 @@ public abstract class AbstractImExportServiceImpl implements ImExportService {
@Override
public void exportConfig(OutputStream os, List<Long> configList) {
var monitorList = configList.stream().map(it -> monitorService.getMonitorDto(it)).filter(Objects::nonNull).map(this::convert).toList();
var monitorList = configList.stream()
.map(monitorService::getMonitorDtoForExport)
.filter(Objects::nonNull)
.map(this::convert)
.toList();
writeOs(monitorList, os);
}
@@ -168,7 +172,7 @@ public abstract class AbstractImExportServiceImpl implements ImExportService {
param.setType(it.type);
param.setParamValue(it.value);
return param;
}).toList());
}).toList(), false);
} else {
monitorDto.setParams(Collections.emptyList());
}
@@ -43,6 +43,7 @@ import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.support.event.MonitorDeletedEvent;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.common.util.JexlCheckerUtil;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
@@ -103,6 +104,10 @@ public class MonitorServiceImpl implements MonitorService {
public static final String PATTERN_HTTPS = "(?i)https://";
private static final Long MONITOR_ID_TMP = 1000000000L;
private static final byte ALL_MONITOR_STATUS = 9;
private static final Map<String, Set<String>> CREDENTIAL_DESTINATION_FIELDS = Map.of(
"apiKey", Set.of("host", "port", "ssl"),
"__sd_token__", Set.of("__sd_url__", "__sd_authType__"),
"__sd_password__", Set.of("__sd_url__", "__sd_authType__"));
public static final String PARAM_FIELD_PORT = "port";
@Autowired
@@ -323,10 +328,33 @@ public class MonitorServiceImpl implements MonitorService {
monitorDto.setCollector(null);
}
// Parameter definition structure verification
List<ParamDefineInfo> paramDefines = appService.getAppParamDefines(monitor.getApp());
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
List<ParamDefineInfo> paramDefines = new ArrayList<>();
List<ParamDefineInfo> applicationParamDefines = appService.getAppParamDefines(monitor.getApp());
if (!CollectionUtils.isEmpty(applicationParamDefines)) {
paramDefines.addAll(applicationParamDefines);
}
if (!isStatic && !Objects.equals(monitor.getApp(), monitor.getScrape())) {
List<ParamDefineInfo> scrapeParamDefines = appService.getAppParamDefines(monitor.getScrape());
if (!CollectionUtils.isEmpty(scrapeParamDefines)) {
paramDefines.addAll(scrapeParamDefines);
}
}
boolean restoresMaskedCredential = (Boolean.TRUE.equals(isModify)
|| (isModify == null && monitor.getId() != null))
&& !CollectionUtils.isEmpty(paramDefines)
&& paramDefines.stream()
.filter(paramDefine -> "password".equals(paramDefine.getType()))
.map(ParamDefineInfo::getField)
.map(paramMap::get)
.filter(Objects::nonNull)
.anyMatch(param -> MonitorParam.isSecretMask(param.getParamValue()));
Map<String, Param> storedParams = restoresMaskedCredential
? paramDao.findParamsByMonitorId(monitor.getId()).stream()
.collect(Collectors.toMap(Param::getField, param -> param))
: Map.of();
if (!CollectionUtils.isEmpty(paramDefines)) {
boolean isStatic = CommonConstants.SCRAPE_STATIC.equals(monitor.getScrape())
|| !StringUtils.hasText(monitor.getScrape());
for (ParamDefineInfo paramDefine : paramDefines) {
String field = paramDefine.getField();
MonitorParam param = paramMap.get(field);
@@ -334,6 +362,27 @@ public class MonitorServiceImpl implements MonitorService {
if (!isStatic && "host".equals(field)) {
continue;
}
if ("password".equals(paramDefine.getType())
&& param != null
&& MonitorParam.isSecretMask(param.getParamValue())) {
if (!restoresMaskedCredential) {
throw new IllegalArgumentException("The credential mask cannot be used as a new value.");
}
Param storedParam = storedParams.get(field);
if (storedParam == null || !StringUtils.hasText(storedParam.getParamValue())) {
throw new IllegalArgumentException("The masked credential has no stored value.");
}
String storedValue = storedParam.getParamValue();
boolean currentCiphertext = AesUtil.isCiphertext(storedValue);
boolean legacyCiphertext = !AesUtil.DEFAULT_ENCODE_RULES.equals(AesUtil.getDefaultSecretKey())
&& AesUtil.isCiphertext(storedValue, AesUtil.DEFAULT_ENCODE_RULES);
if (!currentCiphertext && !legacyCiphertext) {
throw new IllegalStateException("The stored credential migration is incomplete.");
}
validateCredentialDestination(field, paramMap, storedParams);
param.setParamValue(storedValue);
param.setType(CommonConstants.PARAM_TYPE_PASSWORD);
}
if (paramDefine.isRequired() && (param == null || param.getParamValue() == null)) {
throw new IllegalArgumentException("Params field " + field + " is required.");
}
@@ -342,7 +391,27 @@ public class MonitorServiceImpl implements MonitorService {
}
}
}
checkJobFields(monitorDto.getMonitor().getApp());
checkJobFields(monitor.getApp());
if (!isStatic && !Objects.equals(monitor.getApp(), monitor.getScrape())) {
checkJobFields(monitor.getScrape());
}
}
private void validateCredentialDestination(
String credentialField,
Map<String, MonitorParam> submittedParams,
Map<String, Param> storedParams) {
for (String destinationField : CREDENTIAL_DESTINATION_FIELDS.getOrDefault(
credentialField, Set.of())) {
MonitorParam submitted = submittedParams.get(destinationField);
Param stored = storedParams.get(destinationField);
String submittedValue = submitted == null ? null : submitted.getParamValue();
String storedValue = stored == null ? null : stored.getParamValue();
if (!Objects.equals(submittedValue, storedValue)) {
throw new IllegalArgumentException(
"A credential must be re-entered when its destination changes.");
}
}
}
private void checkJobFields(String app) {
@@ -509,6 +578,16 @@ public class MonitorServiceImpl implements MonitorService {
@Override
@Transactional(readOnly = true)
public MonitorDto getMonitorDto(long id) throws RuntimeException {
return getMonitorDto(id, true);
}
@Override
@Transactional(readOnly = true)
public MonitorDto getMonitorDtoForExport(long id) throws RuntimeException {
return getMonitorDto(id, false);
}
private MonitorDto getMonitorDto(long id, boolean maskCredentials) throws RuntimeException {
Optional<Monitor> monitorOptional = monitorDao.findById(id);
if (monitorOptional.isPresent()) {
// Get current user ID for favorite status
@@ -524,7 +603,7 @@ public class MonitorServiceImpl implements MonitorService {
Monitor monitor = monitorOptional.get();
MonitorDto monitorDto = new MonitorDto();
List<Param> params = paramDao.findParamsByMonitorId(id);
monitorDto.setParams(params);
monitorDto.setParams(params, maskCredentials);
List<MetricsInfo> metricsInfos;
if (DispatchConstants.PROTOCOL_PROMETHEUS.equalsIgnoreCase(monitor.getApp())
|| monitor.getType() == CommonConstants.MONITOR_TYPE_PUSH_AUTO_CREATE) {
@@ -62,7 +62,7 @@ params:
en-US: Access Token
ja-JP: アクセストークン
# type-param field type(most mapping the html input type)
type: text
type: password
# dependent parameter values list
depend:
__sd_authType__:
@@ -65,7 +65,7 @@ params:
zh-CN: API Key
en-US: API Key
ja-JP: API キー
type: text
type: password
required: false
# collect metrics config list
metrics:
@@ -72,4 +72,24 @@ class PasswordParamValidatorTest {
assertEquals(ciphertext, param.getParamValue());
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, param.getType());
}
@Test
void validate_IgnoresLegacyCiphertextUntilMigrationReencryptsIt() {
String originalSecret = AesUtil.getDefaultSecretKey();
try {
AesUtil.setDefaultSecretKey("nextSecretKey123");
String legacyCiphertext = AesUtil.aesEncode("password123", AesUtil.DEFAULT_ENCODE_RULES);
ParamDefineInfo paramDefine = new ParamDefineInfo();
paramDefine.setType("password");
MonitorParam param = new MonitorParam();
param.setParamValue(legacyCiphertext);
validator.validate(paramDefine, param);
assertEquals(legacyCiphertext, param.getParamValue());
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, param.getType());
} finally {
AesUtil.setDefaultSecretKey(originalSecret);
}
}
}
@@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Map;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.util.AesUtil;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
class ServiceCredentialMigrationTest {
private static final String TEST_SECRET = "0123456789abcdef";
private JdbcTemplate jdbcTemplate;
@BeforeEach
void setUp() {
AesUtil.setDefaultSecretKey(TEST_SECRET);
DriverManagerDataSource dataSource = new DriverManagerDataSource(
"jdbc:h2:mem:credential-migration;MODE=MySQL;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE",
"sa",
"");
jdbcTemplate = new JdbcTemplate(dataSource);
jdbcTemplate.execute("DROP ALL OBJECTS");
jdbcTemplate.execute("""
CREATE TABLE hzb_monitor (
id BIGINT PRIMARY KEY,
app VARCHAR(100),
scrape VARCHAR(100)
)
""");
jdbcTemplate.execute("""
CREATE TABLE hzb_param (
id BIGINT PRIMARY KEY,
monitor_id BIGINT,
field VARCHAR(100),
param_value VARCHAR(8126),
type TINYINT
)
""");
jdbcTemplate.execute("""
CREATE TABLE hzb_config (
type VARCHAR(100) PRIMARY KEY,
content VARCHAR(8192)
)
""");
}
@AfterEach
void tearDown() {
AesUtil.setDefaultSecretKey(AesUtil.DEFAULT_ENCODE_RULES);
}
@Test
void migratesLegacyCredentialsInTheDatabaseIdempotently() {
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 1L, "ollama", "static");
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 2L, "linux", "http_sd");
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 3L, "website", "static");
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)", 4L, "ollama", "static");
String alreadyEncrypted = AesUtil.aesEncode("existing-ciphertext");
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
11L, 1L, "apiKey", "legacy-ollama-key", CommonConstants.PARAM_TYPE_STRING);
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
12L, 2L, "__sd_token__", "legacy-http-sd-token", CommonConstants.PARAM_TYPE_STRING);
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
13L, 3L, "apiKey", "ordinary-text", CommonConstants.PARAM_TYPE_STRING);
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
14L, 4L, "apiKey", alreadyEncrypted, CommonConstants.PARAM_TYPE_PASSWORD);
ServiceCredentialMigration migration = new ServiceCredentialMigration(jdbcTemplate);
assertEquals(2, migration.migrateStoredCredentials());
Map<String, Object> ollama = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 11");
Map<String, Object> httpSd = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 12");
Map<String, Object> ordinary = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 13");
Map<String, Object> encrypted = jdbcTemplate.queryForMap(
"SELECT param_value, type FROM hzb_param WHERE id = 14");
String ollamaCiphertext = String.valueOf(ollama.get("PARAM_VALUE"));
String httpSdCiphertext = String.valueOf(httpSd.get("PARAM_VALUE"));
assertNotEquals("legacy-ollama-key", ollamaCiphertext);
assertNotEquals("legacy-http-sd-token", httpSdCiphertext);
assertTrue(AesUtil.isCiphertext(ollamaCiphertext));
assertTrue(AesUtil.isCiphertext(httpSdCiphertext));
assertEquals("legacy-ollama-key", AesUtil.aesDecode(ollamaCiphertext));
assertEquals("legacy-http-sd-token", AesUtil.aesDecode(httpSdCiphertext));
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, ((Number) ollama.get("TYPE")).byteValue());
assertEquals(CommonConstants.PARAM_TYPE_PASSWORD, ((Number) httpSd.get("TYPE")).byteValue());
assertEquals("ordinary-text", ordinary.get("PARAM_VALUE"));
assertEquals(CommonConstants.PARAM_TYPE_STRING, ((Number) ordinary.get("TYPE")).byteValue());
assertEquals(alreadyEncrypted, encrypted.get("PARAM_VALUE"));
assertEquals(0, migration.migrateStoredCredentials());
assertEquals(ollamaCiphertext, jdbcTemplate.queryForObject(
"SELECT param_value FROM hzb_param WHERE id = 11", String.class));
}
@Test
void startupMigrationRunsOnlyOnceAndDoesNotRewriteLaterCredentials() {
jdbcTemplate.update("INSERT INTO hzb_monitor(id, app, scrape) VALUES (?, ?, ?)",
1L, "ollama", "static");
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
11L, 1L, "apiKey", "legacy-ollama-key", CommonConstants.PARAM_TYPE_STRING);
ServiceCredentialMigration migration = new ServiceCredentialMigration(jdbcTemplate);
migration.run();
jdbcTemplate.update("INSERT INTO hzb_param VALUES (?, ?, ?, ?, ?)",
12L, 1L, "apiKey", "later-operator-value", CommonConstants.PARAM_TYPE_STRING);
migration.run();
assertTrue(AesUtil.isCiphertext(jdbcTemplate.queryForObject(
"SELECT param_value FROM hzb_param WHERE id = 11", String.class)));
assertEquals("later-operator-value", jdbcTemplate.queryForObject(
"SELECT param_value FROM hzb_param WHERE id = 12", String.class));
assertEquals(1, jdbcTemplate.queryForObject(
"SELECT COUNT(*) FROM hzb_config WHERE type = 'migration.service-credentials.v1'",
Integer.class));
}
}
@@ -123,13 +123,18 @@ class MonitorControllerTest {
MonitorDto monitorDto = new MonitorDto();
monitorDto.setMonitor(monitor);
Param secret = new Param();
secret.setField("apiKey");
secret.setParamValue("ciphertext-must-not-leave-the-api");
secret.setType(CommonConstants.PARAM_TYPE_PASSWORD);
monitorDto.setParams(List.of(secret));
Mockito.when(monitorService.getMonitorDto(6565463543L))
.thenReturn(monitorDto);
this.mockMvc.perform(MockMvcRequestBuilders.get("/api/monitor/{id}", 6565463543L))
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.data.params[0].paramValue").value("******"))
.andExpect(status().isOk())
.andReturn();
}
@@ -46,6 +46,7 @@ import java.util.Map;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -85,6 +86,12 @@ class AppServiceTest {
assertDoesNotThrow(() -> appService.getAppParamDefines("jvm"));
}
@Test
void credentialParametersUsePasswordType() {
assertEquals("password", findParam("ollama", "apiKey").getType());
assertEquals("password", findParam("http_sd", "__sd_token__").getType());
}
@Test
void getAppParamDefinesShouldNotContainDuplicateFields() {
List<ParamDefineInfo> paramDefines = appService.getAppParamDefines("nvidia");
@@ -203,4 +210,11 @@ class AppServiceTest {
assertNotNull(appParamDefines);
assertTrue(appParamDefines.stream().anyMatch(t -> t.getField().equals("host_test")));
}
private ParamDefineInfo findParam(String app, String field) {
return appService.getAppParamDefines(app).stream()
.filter(param -> field.equals(param.getField()))
.findFirst()
.orElseThrow();
}
}
@@ -32,6 +32,8 @@ import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.manager.config.ManagerSseManager;
import org.apache.hertzbeat.manager.service.impl.AbstractImExportServiceImpl;
import org.apache.hertzbeat.manager.service.impl.JsonImExportServiceImpl;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.pojo.dto.MonitorParam;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -110,6 +112,41 @@ class JsonImExportServiceTest {
assertEquals("JSON", jsonImExportService.type());
}
@Test
void testExportConfigPreservesEncryptedCredentialForImportRoundTrip() {
String ciphertext = "HBA2-export-ciphertext";
MonitorDto monitorDto = new MonitorDto();
monitorDto.setMonitor(Monitor.builder().id(1L).name("ollama").app("ollama").build());
MonitorParam secret = new MonitorParam();
secret.setField("apiKey");
secret.setType(org.apache.hertzbeat.common.constants.CommonConstants.PARAM_TYPE_PASSWORD);
secret.setParamValue(ciphertext);
monitorDto.setParamInfos(List.of(secret));
org.mockito.Mockito.when(monitorService.getMonitorDtoForExport(1L)).thenReturn(monitorDto);
ByteArrayOutputStream output = new ByteArrayOutputStream();
jsonImExportService.exportConfig(output, List.of(1L));
assertTrue(output.toString(StandardCharsets.UTF_8).contains(ciphertext));
}
@Test
void testImportConfigPreservesEncryptedCredentialForImportRoundTrip() {
String ciphertext = "HBA2-import-ciphertext";
String json = "[{\"monitor\":{\"name\":\"ollama-import\",\"app\":\"ollama\","
+ "\"intervals\":6000,\"status\":1},\"params\":[{\"field\":\"apiKey\","
+ "\"type\":2,\"value\":\"" + ciphertext + "\"}]}]";
ArgumentCaptor<List<Param>> paramsCaptor = ArgumentCaptor.forClass(List.class);
doNothing().when(monitorService).addMonitor(
org.mockito.Mockito.any(), paramsCaptor.capture(),
org.mockito.Mockito.any(), org.mockito.Mockito.any());
jsonImExportService.importConfig(
"ollama.json", new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)));
assertEquals(ciphertext, paramsCaptor.getValue().get(0).getParamValue());
}
@Test
void testImportConfig_shouldSetInstanceFromHostAndPortParams() {
String json = "[{\"monitor\":{\"name\":\"test\",\"app\":\"windows\",\"intervals\":6000,\"status\":1},"
@@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
@@ -42,6 +44,7 @@ import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorBindDao;
@@ -49,6 +52,7 @@ import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.pojo.dto.AppCount;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.pojo.dto.MonitorParam;
import org.apache.hertzbeat.manager.pojo.dto.ParamDefineInfo;
import org.apache.hertzbeat.manager.scheduler.CollectJobScheduling;
import org.apache.hertzbeat.manager.component.validator.ParamValidatorManager;
@@ -161,6 +165,242 @@ class MonitorServiceTest {
return paramDefine;
}
@Test
void validateModifyRestoresTheStoredCredentialBehindTheResponseMask() {
long monitorId = 101L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("ollama-local")
.app("ollama")
.scrape("static")
.instance("127.0.0.1")
.intervals(60)
.build();
Param submitted = Param.builder()
.monitorId(monitorId)
.field("apiKey")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
String storedCiphertext = AesUtil.aesEncode("stored-ollama-key");
Param stored = Param.builder()
.id(11L)
.monitorId(monitorId)
.field("apiKey")
.paramValue(storedCiphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submitted));
when(appService.getAppParamDefines("ollama"))
.thenReturn(List.of(newParamDefine("apiKey", "password", false)));
when(appService.getAppDefineOption("ollama-local")).thenReturn(Optional.empty());
when(monitorDao.findMonitorByNameEquals("ollama-local")).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(stored));
Job job = new Job();
job.setMetrics(Collections.emptyList());
when(appService.getAppDefine("ollama")).thenReturn(job);
monitorService.validate(dto, true);
ArgumentCaptor<MonitorParam> paramCaptor = ArgumentCaptor.forClass(MonitorParam.class);
verify(paramValidatorManager).validate(any(ParamDefineInfo.class), paramCaptor.capture());
assertEquals(storedCiphertext, paramCaptor.getValue().getParamValue());
assertEquals(storedCiphertext, dto.getParams().get(0).getParamValue());
}
@Test
void validateDetectRestoresStoredCredentialForAnExistingMonitor() {
long monitorId = 103L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("ollama-detect")
.app("ollama")
.scrape("static")
.instance("127.0.0.1")
.intervals(60)
.build();
Param submitted = Param.builder()
.monitorId(monitorId)
.field("apiKey")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
String storedCiphertext = AesUtil.aesEncode("stored-detect-key");
Param stored = Param.builder()
.id(31L)
.monitorId(monitorId)
.field("apiKey")
.paramValue(storedCiphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submitted));
when(appService.getAppParamDefines("ollama"))
.thenReturn(List.of(newParamDefine("apiKey", "password", false)));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(stored));
Job job = new Job();
job.setMetrics(Collections.emptyList());
when(appService.getAppDefine("ollama")).thenReturn(job);
monitorService.validate(dto, null);
assertEquals(storedCiphertext, dto.getParams().get(0).getParamValue());
verify(paramValidatorManager).validate(any(ParamDefineInfo.class), any(MonitorParam.class));
}
@Test
void validateModifyRejectsMaskedCredentialWhenDestinationChanges() {
long monitorId = 102L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("ollama-remote")
.app("ollama")
.scrape("static")
.instance("attacker.example")
.intervals(60)
.build();
Param submittedHost = Param.builder()
.monitorId(monitorId)
.field("host")
.paramValue("attacker.example")
.type(CommonConstants.PARAM_TYPE_STRING)
.build();
Param submittedSecret = Param.builder()
.monitorId(monitorId)
.field("apiKey")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
Param storedHost = Param.builder()
.id(21L)
.monitorId(monitorId)
.field("host")
.paramValue("trusted.example")
.type(CommonConstants.PARAM_TYPE_STRING)
.build();
Param storedSecret = Param.builder()
.id(22L)
.monitorId(monitorId)
.field("apiKey")
.paramValue(AesUtil.aesEncode("stored-ollama-key"))
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submittedHost, submittedSecret));
when(appService.getAppParamDefines("ollama")).thenReturn(List.of(
newParamDefine("host", "host", true),
newParamDefine("apiKey", "password", false)));
when(appService.getAppDefineOption("ollama-remote")).thenReturn(Optional.empty());
when(monitorDao.findMonitorByNameEquals("ollama-remote")).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(storedHost, storedSecret));
assertThrows(IllegalArgumentException.class, () -> monitorService.validate(dto, true));
}
@Test
void validateModifyRestoresHttpServiceDiscoveryCredentialFromScrapeDefinition() {
long monitorId = 103L;
Monitor monitor = Monitor.builder()
.id(monitorId)
.name("discovered-prometheus")
.app("prometheus")
.scrape("http_sd")
.instance("https://discovery.example")
.intervals(60)
.build();
Param submittedUrl = Param.builder()
.monitorId(monitorId)
.field("__sd_url__")
.paramValue("https://discovery.example")
.type(CommonConstants.PARAM_TYPE_STRING)
.build();
Param submittedToken = Param.builder()
.monitorId(monitorId)
.field("__sd_token__")
.paramValue(MonitorParam.SECRET_MASK)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
String storedCiphertext = AesUtil.aesEncode("stored-discovery-token");
Param storedUrl = submittedUrl.clone();
Param storedToken = Param.builder()
.id(32L)
.monitorId(monitorId)
.field("__sd_token__")
.paramValue(storedCiphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submittedUrl, submittedToken));
when(appService.getAppParamDefines("prometheus")).thenReturn(Collections.emptyList());
when(appService.getAppParamDefines("http_sd")).thenReturn(List.of(
newParamDefine("__sd_url__", "text", true),
newParamDefine("__sd_token__", "password", false)));
when(appService.getAppDefineOption("discovered-prometheus")).thenReturn(Optional.empty());
when(monitorDao.findMonitorByNameEquals("discovered-prometheus")).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(monitorId)).thenReturn(List.of(storedUrl, storedToken));
Job applicationJob = new Job();
applicationJob.setMetrics(Collections.emptyList());
when(appService.getAppDefine("prometheus")).thenReturn(applicationJob);
Job job = new Job();
job.setMetrics(Collections.emptyList());
when(appService.getAppDefine("http_sd")).thenReturn(job);
monitorService.validate(dto, true);
assertEquals(storedCiphertext, dto.getParams().stream()
.filter(param -> "__sd_token__".equals(param.getField()))
.findFirst()
.orElseThrow()
.getParamValue());
}
@Test
void validateServiceDiscoveryMonitorChecksApplicationAndScrapeCredentials() {
Monitor monitor = Monitor.builder()
.id(104L)
.name("discovered-mysql")
.app("mysql")
.scrape("http_sd")
.instance("https://discovery.example")
.intervals(60)
.build();
Param submittedPassword = Param.builder()
.monitorId(monitor.getId())
.field("password")
.paramValue("database-secret")
.build();
Param submittedDiscoveryToken = Param.builder()
.monitorId(monitor.getId())
.field("__sd_token__")
.paramValue("discovery-secret")
.build();
MonitorDto dto = new MonitorDto();
dto.setMonitor(monitor);
dto.setParams(List.of(submittedPassword, submittedDiscoveryToken));
ParamDefineInfo applicationPassword = newParamDefine("password", "password", true);
ParamDefineInfo discoveryToken = newParamDefine("__sd_token__", "password", true);
when(appService.getAppParamDefines("mysql")).thenReturn(List.of(applicationPassword));
when(appService.getAppParamDefines("http_sd")).thenReturn(List.of(discoveryToken));
Job applicationJob = new Job();
applicationJob.setMetrics(Collections.emptyList());
Job scrapeJob = new Job();
scrapeJob.setMetrics(Collections.emptyList());
when(appService.getAppDefine("mysql")).thenReturn(applicationJob);
when(appService.getAppDefine("http_sd")).thenReturn(scrapeJob);
monitorService.validate(dto, null);
verify(paramValidatorManager).validate(
eq(applicationPassword), argThat(param -> "password".equals(param.getField())));
verify(paramValidatorManager).validate(
eq(discoveryToken), argThat(param -> "__sd_token__".equals(param.getField())));
}
@Test
void detectMonitorEmpty() {
Monitor monitor = Monitor.builder()
@@ -775,6 +1015,38 @@ class MonitorServiceTest {
assertNotNull(monitorDto);
}
@Test
void getMonitorDtoMasksCredentialsButExportKeepsCiphertext() {
long id = 2L;
Monitor monitor = Monitor.builder()
.jobId(id)
.intervals(1)
.app("ollama")
.name("ollama-export")
.instance("localhost")
.id(id)
.build();
String ciphertext = AesUtil.aesEncode("portable-secret");
Param secret = Param.builder()
.monitorId(id)
.field("apiKey")
.paramValue(ciphertext)
.type(CommonConstants.PARAM_TYPE_PASSWORD)
.build();
when(monitorDao.findById(id)).thenReturn(Optional.of(monitor));
when(paramDao.findParamsByMonitorId(id)).thenReturn(List.of(secret));
Job job = new Job();
job.setMetrics(new ArrayList<>());
when(appService.getAppDefine(monitor.getApp())).thenReturn(job);
when(collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(id)).thenReturn(Optional.empty());
MonitorDto response = monitorService.getMonitorDto(id);
MonitorDto export = monitorService.getMonitorDtoForExport(id);
assertEquals(MonitorParam.SECRET_MASK, response.getParamInfos().getFirst().getParamValue());
assertEquals(ciphertext, export.getParamInfos().getFirst().getParamValue());
}
@Test
void getMonitors() {
when(monitorDao.findAll(any(Specification.class), any(PageRequest.class))).thenAnswer((invocation) -> {
+12
View File
@@ -50,6 +50,18 @@ You need to provide or develop an HTTP API that meets the following requirements
| Collection interval | Interval time of monitor periodic data collection, unit: second, and the minimum interval that can be set is 30 seconds. |
| Description remarks | For more information about identifying and describing this monitoring, users can note information here. |
### Credential upgrade behavior
The access token and password parameters are encrypted before they are stored
and are returned by the monitor API as `******`. Submitting that mask while
editing an existing monitor keeps the stored credential unchanged. If the
service-discovery URL or authentication type changes, the credential must be
re-entered so that a stored value cannot be replayed to a new endpoint.
On the first startup after upgrading, HertzBeat encrypts access tokens created
by older versions before scheduling service-discovery jobs. The migration is
idempotent and leaves already encrypted values unchanged.
### Usage Steps
1. **Prepare HTTP API**
+11
View File
@@ -28,6 +28,17 @@ allow external access.
| Bound Tags | Tags for categorizing and managing monitoring resources. |
| Description/Remarks | Additional remarks to identify and describe this monitoring. Users can add notes here. |
### Credential upgrade behavior
The API key is encrypted before it is stored and is returned by the monitor API
as `******`. Submitting that mask while editing an existing monitor keeps the
stored key unchanged. If the host, port, or SSL setting changes, the key must
be re-entered so that a stored credential cannot be replayed to a new endpoint.
On the first startup after upgrading, HertzBeat encrypts API keys created by
older versions before scheduling collection jobs. The migration is idempotent
and does not change an already encrypted value.
### Collection Metrics
#### Metric Set: Version Info