Merge branch 'master' into fix/008-questdb-sql-injection

This commit is contained in:
aias00
2026-08-17 13:50:00 +08:00
committed by GitHub
154 changed files with 5014 additions and 405 deletions
@@ -37,6 +37,7 @@ concurrency:
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
@@ -53,3 +54,6 @@ jobs:
- name: EsLint Test
working-directory: web-app
run: pnpm lint:ts
- name: Unit Test
working-directory: web-app
run: pnpm test
@@ -78,24 +78,32 @@ public class ConversationServiceImpl implements ConversationService {
.build());
}
log.info("Starting streaming conversation: {}", conversationId);
ChatConversation conversation = conversationDao.findById(conversationId)
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
ChatConversation conversation;
if (conversationId == null) {
// The API contract makes conversationId optional, so create a conversation for the first message.
conversation = new ChatConversation();
conversation.setTitle(buildConversationTitle(message));
conversation = conversationDao.save(conversation);
} else {
conversation = conversationDao.findById(conversationId)
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
}
Long currentConversationId = conversation.getId();
log.info("Starting streaming conversation: {}", currentConversationId);
// Manually load messages for conversation history
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId);
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(currentConversationId);
conversation.setMessages(messages);
if (conversation.getTitle().startsWith("conversation")) {
// Auto-generate title from first user message
String title = message.length() > 30 ? message.substring(0, 27) + "..." : message;
conversation.setTitle(title);
conversation.setTitle(buildConversationTitle(message));
conversationDao.save(conversation);
}
// Add user message to conversation
ChatMessage chatMessage = ChatMessage.builder()
.conversationId(conversationId)
.conversationId(currentConversationId)
.content(message)
.role("user")
.build();
@@ -103,7 +111,7 @@ public class ConversationServiceImpl implements ConversationService {
ChatRequestContext context = ChatRequestContext.builder()
.message(message)
.conversationId(conversationId)
.conversationId(currentConversationId)
.conversationHistory(messages)
.build();
@@ -116,7 +124,7 @@ public class ConversationServiceImpl implements ConversationService {
.map(chunk -> {
fullResponse.append(chunk);
ChatResponseChunk responseChunk = ChatResponseChunk.builder()
.conversationId(conversationId)
.conversationId(currentConversationId)
.userMessageId(finalChatMessage.getId())
.response(chunk)
.build();
@@ -128,13 +136,13 @@ public class ConversationServiceImpl implements ConversationService {
.concatWith(Flux.defer(() -> {
// Add the complete AI response to conversation
ChatMessage assistantMessage = ChatMessage.builder()
.conversationId(conversationId)
.conversationId(currentConversationId)
.content(fullResponse.toString())
.role("assistant")
.build();
assistantMessage = messageDao.save(assistantMessage);
ChatResponseChunk finalResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.conversationId(currentConversationId)
.response("")
.assistantMessageId(assistantMessage.getId())
.build();
@@ -143,12 +151,12 @@ public class ConversationServiceImpl implements ConversationService {
.event("complete")
.build());
}))
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", conversationId))
.doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", conversationId,
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", currentConversationId))
.doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", currentConversationId,
error.getMessage(), error))
.onErrorResume(error -> {
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
.conversationId(conversationId)
.conversationId(currentConversationId)
.response("An error occurred: " + error.getMessage())
.userMessageId(finalChatMessage.getId())
.build();
@@ -165,6 +173,10 @@ public class ConversationServiceImpl implements ConversationService {
return conversationDao.save(conversation);
}
private String buildConversationTitle(String message) {
return message.length() > 30 ? message.substring(0, 27) + "..." : message;
}
@Override
public ChatConversation getConversation(Long conversationId) {
if (conversationId == null) {
@@ -19,17 +19,23 @@ package org.apache.hertzbeat.ai.tools.impl;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.service.SopScheduleService;
import org.apache.hertzbeat.ai.sop.model.SopDefinition;
import org.apache.hertzbeat.ai.sop.model.SopParameter;
import org.apache.hertzbeat.ai.sop.registry.SkillRegistry;
import org.apache.hertzbeat.ai.utils.SopMessageUtil;
import org.apache.hertzbeat.ai.tools.ScheduleTools;
import org.apache.hertzbeat.common.entity.ai.SopSchedule;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import tools.jackson.core.type.TypeReference;
/**
* Implementation of ScheduleTools for AI-driven schedule management.
@@ -78,18 +84,22 @@ public class ScheduleToolsImpl implements ScheduleTools {
@Tool(name = "createScheduleWithConversation",
description = "Create a scheduled task for a specific conversation. "
+ "Use the conversationId from the system context. "
+ "Pass skill parameters as a JSON object when the skill requires inputs. "
+ "The cron expression should be in 6-digit Spring format.")
public String createScheduleWithConversation(
@ToolParam(description = "Conversation ID from the system context", required = true) Long conversationId,
@ToolParam(description = "Name of the skill to schedule (e.g., 'daily_inspection')", required = true) String skillName,
@ToolParam(description = "Cron expression in Spring format (e.g., '0 0 9 * * ?')", required = true) String cronExpression,
@ToolParam(description = "Description of the schedule", required = false) String description) {
@ToolParam(description = "Description of the schedule", required = false) String description,
@ToolParam(description = "Skill parameters as a JSON object (e.g., '{\"monitorId\":123}')",
required = false) String paramsJson) {
log.info("AI creating schedule: conversationId={}, skill={}, cron={}, desc={}",
conversationId, skillName, cronExpression, description);
// Validate skill exists
if (skillRegistry.getSkill(skillName) == null) {
SopDefinition skill = skillRegistry.getSkill(skillName);
if (skill == null) {
String available = String.join(", ",
skillRegistry.getAllSkills().stream()
.map(s -> s.getName())
@@ -103,11 +113,16 @@ public class ScheduleToolsImpl implements ScheduleTools {
}
try {
// Check for duplicate schedule (same skill + cron expression)
Map<String, Object> params = parseSkillParams(paramsJson);
validateRequiredParameters(skill, params);
String serializedParams = params.isEmpty() ? null : JsonUtil.toJson(params);
// Parameters are part of a schedule's identity so the same skill and cron can target different inputs.
List<SopSchedule> existing = scheduleService.getSchedulesByConversation(conversationId);
boolean duplicate = existing.stream()
.anyMatch(s -> s.getSopName().equals(skillName)
&& s.getCronExpression().equals(cronExpression));
.anyMatch(schedule -> Objects.equals(schedule.getSopName(), skillName)
&& Objects.equals(schedule.getCronExpression(), cronExpression)
&& hasSameParams(schedule.getSopParams(), params));
if (duplicate) {
return SopMessageUtil.getMessage("schedule.create.duplicate",
new Object[]{skillName, cronExpression}, null)
@@ -118,6 +133,7 @@ public class ScheduleToolsImpl implements ScheduleTools {
schedule.setConversationId(conversationId);
schedule.setSopName(skillName);
schedule.setCronExpression(cronExpression);
schedule.setSopParams(serializedParams);
schedule.setEnabled(true);
SopSchedule created = scheduleService.createSchedule(schedule);
@@ -150,6 +166,55 @@ public class ScheduleToolsImpl implements ScheduleTools {
}
}
private Map<String, Object> parseSkillParams(String paramsJson) {
if (paramsJson == null || paramsJson.isBlank()) {
return Map.of();
}
Map<String, Object> params;
try {
params = JsonUtil.fromJson(paramsJson, new TypeReference<>() {});
} catch (RuntimeException e) {
throw new IllegalArgumentException("Skill parameters must be a valid JSON object", e);
}
if (params == null) {
throw new IllegalArgumentException("Skill parameters must be a valid JSON object");
}
return params;
}
private void validateRequiredParameters(SopDefinition skill, Map<String, Object> params) {
if (skill.getParameters() == null) {
return;
}
for (SopParameter parameter : skill.getParameters()) {
Object value = params.get(parameter.getName());
if (isMissing(value)) {
value = parameter.getDefaultValue();
}
if (parameter.isRequired() && isMissing(value)) {
throw new IllegalArgumentException(
"Required skill parameter is missing: " + parameter.getName());
}
}
}
private boolean isMissing(Object value) {
return value == null || value instanceof String text && text.isBlank();
}
private boolean hasSameParams(String existingJson, Map<String, Object> params) {
if (existingJson == null || existingJson.isBlank()) {
return params.isEmpty();
}
try {
Map<String, Object> existingParams = JsonUtil.fromJson(existingJson, new TypeReference<>() {});
return Objects.equals(existingParams, params);
} catch (RuntimeException e) {
log.warn("Failed to parse parameters of an existing schedule", e);
return false;
}
}
@Override
@Tool(name = "listSchedulesForConversation",
description = "List all scheduled tasks for a specific conversation. "
@@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import com.usthe.sureness.subject.SubjectSum;
@@ -123,6 +124,47 @@ class ConversationServiceImplTest {
assertEquals(subject, contextCaptor.getValue().getSubject());
}
/**
* The service should create a conversation and return its ID when the client omits the optional conversation ID.
*/
@Test
void streamChatShouldCreateConversationWhenConversationIdIsMissing() {
AtomicLong messageId = new AtomicLong(20L);
when(chatClientProviderService.isConfigured()).thenReturn(true);
when(conversationDao.save(any(ChatConversation.class))).thenAnswer(invocation -> {
ChatConversation savedConversation = invocation.getArgument(0);
savedConversation.setId(CONVERSATION_ID);
return savedConversation;
});
when(messageDao.findByConversationIdOrderByGmtCreateAsc(CONVERSATION_ID)).thenReturn(List.of());
when(messageDao.save(any(ChatMessage.class))).thenAnswer(invocation -> {
ChatMessage savedMessage = invocation.getArgument(0);
savedMessage.setId(messageId.getAndIncrement());
return savedMessage;
});
when(chatClientProviderService.streamChat(any(ChatRequestContext.class)))
.thenReturn(Flux.just("本轮回答"));
List<ServerSentEvent<ChatResponseChunk>> events = conversationService
.streamChat("本轮问题", null)
.collectList()
.block();
assertNotNull(events);
assertEquals(2, events.size());
assertEquals(CONVERSATION_ID, events.get(0).data().getConversationId());
assertEquals(CONVERSATION_ID, events.get(1).data().getConversationId());
ArgumentCaptor<ChatRequestContext> contextCaptor = ArgumentCaptor.forClass(ChatRequestContext.class);
verify(chatClientProviderService).streamChat(contextCaptor.capture());
assertEquals(CONVERSATION_ID, contextCaptor.getValue().getConversationId());
assertEquals(List.of(), contextCaptor.getValue().getConversationHistory());
ArgumentCaptor<ChatConversation> conversationCaptor = ArgumentCaptor.forClass(ChatConversation.class);
verify(conversationDao).save(conversationCaptor.capture());
assertEquals("本轮问题", conversationCaptor.getValue().getTitle());
verifyNoMoreInteractions(conversationDao);
}
/**
* Deleting a conversation must remove its schedules before they can push more messages.
*/
@@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.tools.impl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.List;
import org.apache.hertzbeat.ai.service.SopScheduleService;
import org.apache.hertzbeat.ai.sop.model.SopDefinition;
import org.apache.hertzbeat.ai.sop.model.SopParameter;
import org.apache.hertzbeat.ai.sop.registry.SkillRegistry;
import org.apache.hertzbeat.common.entity.ai.SopSchedule;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Verifies that AI-created SOP schedules validate and persist skill parameters.
*/
@ExtendWith(MockitoExtension.class)
class ScheduleToolsImplTest {
private static final String CRON = "0 0 9 * * ?";
@Mock
private SopScheduleService scheduleService;
@Mock
private SkillRegistry skillRegistry;
private ScheduleToolsImpl scheduleTools;
@BeforeEach
void setUp() {
scheduleTools = new ScheduleToolsImpl(scheduleService, skillRegistry);
}
@Test
void createScheduleShouldPersistSkillParameters() {
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
when(scheduleService.getSchedulesByConversation(7L)).thenReturn(List.of());
when(scheduleService.createSchedule(any())).thenAnswer(invocation -> {
SopSchedule schedule = invocation.getArgument(0);
schedule.setId(9L);
return schedule;
});
String result = scheduleTools.createScheduleWithConversation(
7L, "diagnosis", CRON, "daily diagnosis", "{\"monitorId\":42}");
ArgumentCaptor<SopSchedule> captor = ArgumentCaptor.forClass(SopSchedule.class);
verify(scheduleService).createSchedule(captor.capture());
assertEquals("{\"monitorId\":42}", captor.getValue().getSopParams());
assertTrue(result.contains("9"));
}
@Test
void createScheduleShouldAllowDifferentParametersAtTheSameTime() {
SopSchedule existing = SopSchedule.builder()
.sopName("diagnosis")
.cronExpression(CRON)
.sopParams("{\"monitorId\":41}")
.build();
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
when(scheduleService.getSchedulesByConversation(7L)).thenReturn(List.of(existing));
when(scheduleService.createSchedule(any())).thenAnswer(invocation -> invocation.getArgument(0));
scheduleTools.createScheduleWithConversation(
7L, "diagnosis", CRON, null, "{\"monitorId\":42}");
verify(scheduleService).createSchedule(any());
}
@Test
void createScheduleShouldRejectMissingRequiredParameter() {
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
String result = scheduleTools.createScheduleWithConversation(
7L, "diagnosis", CRON, null, "{}");
assertTrue(result.contains("monitorId"));
verify(scheduleService, never()).createSchedule(any());
}
@Test
void createScheduleShouldRejectInvalidParameterJson() {
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
String result = scheduleTools.createScheduleWithConversation(
7L, "diagnosis", CRON, null, "not-json");
assertTrue(result.contains("valid JSON object"));
verify(scheduleService, never()).createSchedule(any());
}
private SopDefinition parameterizedSkill() {
SopParameter monitorId = SopParameter.builder()
.name("monitorId")
.required(true)
.build();
return SopDefinition.builder()
.name("diagnosis")
.parameters(List.of(monitorId))
.build();
}
}
@@ -251,20 +251,14 @@ public class MetricsRealTimeAlertCalculator {
}
final int fieldType = field.getType();
// strict jexl aborts the whole rule on undefined variables,
// so define every field even when its value is empty or unparseable
if (fieldType == CommonConstants.TYPE_NUMBER) {
final Double doubleValue;
if ((doubleValue = CommonUtil.parseStrDouble(valueStr)) != null) {
fieldValueMap.put(field.getName(), doubleValue);
}
fieldValueMap.put(field.getName(), CommonUtil.parseStrDouble(valueStr));
} else if (fieldType == CommonConstants.TYPE_TIME) {
final Integer integerValue;
if ((integerValue = CommonUtil.parseStrInteger(valueStr)) != null) {
fieldValueMap.put(field.getName(), integerValue);
}
fieldValueMap.put(field.getName(), CommonUtil.parseStrInteger(valueStr));
} else {
if (StringUtils.isNotEmpty(valueStr)) {
fieldValueMap.put(field.getName(), valueStr);
}
fieldValueMap.put(field.getName(), valueStr);
}
if (field.getLabel()) {
@@ -51,4 +51,6 @@ public class AlertDefineDTO {
private String template;
@Excel(name = "Enable")
private Boolean enable;
@Excel(name = "Datasource")
private String datasource;
}
@@ -0,0 +1,142 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.dto;
import com.fasterxml.jackson.annotation.JsonAlias;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Alibaba Cloud Monitor 2.0 webhook alert entity.
*
* @see <a href="https://help.aliyun.com/zh/cms/cloudmonitor-2-0/notification-object">
* Alibaba Cloud Monitor webhook payload fields</a>
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class AlibabaCloudCmsExternAlert {
private String specversion;
private String id;
private String type;
private String subtype;
private String source;
private String sourcetype;
private String time;
private Long timestamp;
private String subject;
private String datacontenttype;
private String severity;
private String status;
private String userId;
private String ruleId;
private String workspace;
private String traceId;
private String alertMessage;
private String alertEntityId;
private Resource resource;
private Map<String, Object> labels;
private Map<String, Object> annotations;
private AlertData data;
private Map<String, Object> alertEntityFields;
private String ruleUrl;
private String entityUrl;
private String alertRuleUrl;
private String alertHistoryUrl;
/**
* Alert resource.
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public static class Resource {
private Entity entity;
private Map<String, Object> tags;
}
/**
* Alert resource entity.
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public static class Entity {
private String domain;
@JsonAlias("entity_type")
private String entityType;
@JsonAlias("entity_id")
private String entityId;
private Map<String, Object> prop;
}
/**
* Threshold alert data.
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public static class AlertData {
private Object value;
private Object threshold;
private String comparisonOperator;
}
}
@@ -155,6 +155,7 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx
alertDefineDTO.setAnnotations(JsonUtil.fromJson(getCellValueAsString(row.getCell(6)), typeReference));
alertDefineDTO.setTemplate(getCellValueAsString(row.getCell(7)));
alertDefineDTO.setEnable(getCellValueAsBoolean(row.getCell(8)));
alertDefineDTO.setDatasource(getCellValueAsString(row.getCell(9)));
return alertDefineDTO;
}
@@ -186,7 +187,7 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx
CellStyle cellStyle = workbook.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.CENTER);
// set header
String[] headers = {"Name", "Type", "Expr", "Period", "Times", "Labels", "Annotations", "Template", "Enable"};
String[] headers = {"Name", "Type", "Expr", "Period", "Times", "Labels", "Annotations", "Template", "Enable", "Datasource"};
Row headerRow = sheet.createRow(0);
for (int i = 0; i < headers.length; i++) {
Cell cell = headerRow.createCell(i);
@@ -227,6 +228,9 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx
Cell enableCell = row.createCell(8);
enableCell.setCellValue(alertDefineDTO.getEnable());
enableCell.setCellStyle(cellStyle);
Cell datasourceCell = row.createCell(9);
datasourceCell.setCellValue(alertDefineDTO.getDatasource());
datasourceCell.setCellStyle(cellStyle);
}
workbook.write(os);
os.close();
@@ -0,0 +1,195 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service.impl;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.dto.AlibabaCloudCmsExternAlert;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.service.ExternAlertService;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.stereotype.Service;
/**
* Alibaba Cloud Monitor 2.0 external alert service.
*/
@Slf4j
@Service
public class AlibabaCloudCmsExternAlertService implements ExternAlertService {
private static final String SOURCE = "alibabacloud-cms";
private final AlarmCommonReduce alarmCommonReduce;
public AlibabaCloudCmsExternAlertService(AlarmCommonReduce alarmCommonReduce) {
this.alarmCommonReduce = alarmCommonReduce;
}
@Override
public void addExternAlert(String content) {
AlibabaCloudCmsExternAlert externAlert = JsonUtil.fromJson(content, AlibabaCloudCmsExternAlert.class);
if (externAlert == null || StringUtils.isBlank(externAlert.getStatus())) {
log.warn("Failed to parse Alibaba Cloud Monitor external alert content: {}", content);
return;
}
alarmCommonReduce.reduceAndSendAlarm(convert(externAlert));
}
@Override
public String supportSource() {
return SOURCE;
}
private SingleAlert convert(AlibabaCloudCmsExternAlert externAlert) {
boolean resolved = isResolved(externAlert);
long eventTime = getEventTime(externAlert);
return SingleAlert.builder()
.content(getAlertContent(externAlert))
.status(resolved ? CommonConstants.ALERT_STATUS_RESOLVED : CommonConstants.ALERT_STATUS_FIRING)
.startAt(eventTime)
.activeAt(resolved ? null : eventTime)
.endAt(resolved ? eventTime : null)
.labels(buildLabels(externAlert))
.annotations(buildAnnotations(externAlert))
.triggerTimes(1)
.build();
}
private boolean isResolved(AlibabaCloudCmsExternAlert externAlert) {
return "RESOLVED".equalsIgnoreCase(externAlert.getStatus())
|| "RECOVERED".equalsIgnoreCase(externAlert.getStatus())
|| "NORMAL_RESOLVE".equalsIgnoreCase(externAlert.getSubtype());
}
private long getEventTime(AlibabaCloudCmsExternAlert externAlert) {
if (externAlert.getTimestamp() != null && externAlert.getTimestamp() > 0) {
return externAlert.getTimestamp();
}
if (StringUtils.isNotBlank(externAlert.getTime())) {
try {
return Instant.parse(externAlert.getTime()).toEpochMilli();
} catch (DateTimeParseException e) {
log.warn("Failed to parse Alibaba Cloud Monitor event time: {}", externAlert.getTime());
}
}
return Instant.now().toEpochMilli();
}
private Map<String, String> buildLabels(AlibabaCloudCmsExternAlert externAlert) {
Map<String, String> labels = new HashMap<>(16);
putValues(labels, externAlert.getLabels());
AlibabaCloudCmsExternAlert.Resource resource = externAlert.getResource();
if (resource != null) {
putValues(labels, resource.getTags());
AlibabaCloudCmsExternAlert.Entity entity = resource.getEntity();
if (entity != null) {
putIfNotBlank(labels, "resourceDomain", entity.getDomain());
putIfNotBlank(labels, "resourceType", entity.getEntityType());
putIfNotBlank(labels, "resourceId", entity.getEntityId());
}
}
labels.put("__source__", SOURCE);
putIfNotBlank(labels, CommonConstants.LABEL_ALERT_NAME, externAlert.getSubject());
putIfNotBlank(labels, CommonConstants.LABEL_ALERT_SEVERITY, convertSeverity(externAlert.getSeverity()));
putIfNotBlank(labels, "ruleId", externAlert.getRuleId());
putIfNotBlank(labels, "workspace", externAlert.getWorkspace());
putIfNotBlank(labels, "alertEntityId", externAlert.getAlertEntityId());
putIfNotBlank(labels, "userId", externAlert.getUserId());
return labels;
}
private Map<String, String> buildAnnotations(AlibabaCloudCmsExternAlert externAlert) {
Map<String, String> annotations = new HashMap<>(16);
putValues(annotations, externAlert.getAnnotations());
AlibabaCloudCmsExternAlert.Resource resource = externAlert.getResource();
if (resource != null && resource.getEntity() != null) {
putValues(annotations, resource.getEntity().getProp());
}
putValues(annotations, externAlert.getAlertEntityFields());
AlibabaCloudCmsExternAlert.AlertData data = externAlert.getData();
if (data != null) {
putValue(annotations, "value", data.getValue());
putValue(annotations, "threshold", data.getThreshold());
putIfNotBlank(annotations, "comparisonOperator", data.getComparisonOperator());
}
putIfNotBlank(annotations, "alertMessage", externAlert.getAlertMessage());
putIfNotBlank(annotations, "traceId", externAlert.getTraceId());
putIfNotBlank(annotations, "ruleUrl", externAlert.getRuleUrl());
putIfNotBlank(annotations, "entityUrl", externAlert.getEntityUrl());
putIfNotBlank(annotations, "alertRuleUrl", externAlert.getAlertRuleUrl());
putIfNotBlank(annotations, "alertHistoryUrl", externAlert.getAlertHistoryUrl());
return annotations;
}
private String getAlertContent(AlibabaCloudCmsExternAlert externAlert) {
if (StringUtils.isNotBlank(externAlert.getAlertMessage())) {
return externAlert.getAlertMessage();
}
if (StringUtils.isNotBlank(externAlert.getSubject())) {
return externAlert.getSubject();
}
return "Alibaba Cloud Monitor alert";
}
private String convertSeverity(String severity) {
if (StringUtils.isBlank(severity)) {
return null;
}
return switch (severity.toUpperCase(Locale.ROOT)) {
case "EMERGENCY" -> CommonConstants.ALERT_SEVERITY_EMERGENCY;
case "CRITICAL" -> CommonConstants.ALERT_SEVERITY_CRITICAL;
case "WARN", "WARNING" -> CommonConstants.ALERT_SEVERITY_WARNING;
case "INFO", "INFORMATIONAL" -> CommonConstants.ALERT_SEVERITY_INFO;
default -> severity.toLowerCase(Locale.ROOT);
};
}
private void putValues(Map<String, String> target, Map<String, Object> values) {
if (values == null || values.isEmpty()) {
return;
}
values.forEach((key, value) -> putValue(target, key, value));
}
private void putValue(Map<String, String> target, String key, Object value) {
if (StringUtils.isBlank(key) || value == null) {
return;
}
String stringValue;
if (value instanceof Map<?, ?> || value instanceof Collection<?>) {
stringValue = JsonUtil.toJson(value);
} else {
stringValue = String.valueOf(value);
}
putIfNotBlank(target, key, stringValue);
}
private void putIfNotBlank(Map<String, String> target, String key, String value) {
if (StringUtils.isNotBlank(value)) {
target.put(key, value);
}
}
}
@@ -27,11 +27,13 @@ import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
@@ -43,6 +45,7 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -254,4 +257,89 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
}
@Test
void testEmptyStringFieldStillTriggersAlert() throws InterruptedException {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
builder.setId(518679137103104L)
.setApp("fullsite")
.setMetrics("summary")
.setPriority(1)
.setCode(CollectRep.Code.SUCCESS);
CollectRep.Field url = CollectRep.Field.newBuilder().setName("url").setType(CommonConstants.TYPE_STRING).setLabel(true).build();
CollectRep.Field statusCode = CollectRep.Field.newBuilder().setName("statusCode").setType(CommonConstants.TYPE_STRING).build();
CollectRep.Field errorMsg = CollectRep.Field.newBuilder().setName("errorMsg").setType(CommonConstants.TYPE_STRING).build();
Map<String, String> meta = new HashMap<>();
meta.put(MetricDataConstants.INSTANCE_NAME, "site");
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
builder.addMetadataAll(meta);
builder.addAllFields(Lists.newArrayList(url, statusCode, errorMsg));
builder.addValueRow(CollectRep.ValueRow.newBuilder()
.addColumn("https://example.com/broken").addColumn("404").addColumn("").build());
CollectRep.MetricsData metricsData = builder.build();
AlertDefine matchDefine = new AlertDefine();
matchDefine.setId(1L);
matchDefine.setName("sitemap-status");
matchDefine.setExpr("equals(__app__,\"fullsite\") && !matches(statusCode,\"^2[0-9]+\") && !contains(errorMsg,\"timed out\")");
matchDefine.setTemplate("site down: ${url}");
matchDefine.setTimes(1);
when(alertDefineService.getMetricsRealTimeAlertDefines()).thenReturn(Collections.singletonList(matchDefine));
when(dataQueue.pollMetricsDataToAlerter()).thenReturn(metricsData).thenThrow(new InterruptedException());
metricsRealTimeAlertCalculator.startCalculate();
Thread.sleep(3000);
ArgumentCaptor<SingleAlert> alertCaptor = ArgumentCaptor.forClass(SingleAlert.class);
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(alertCaptor.capture());
assertEquals("site down: https://example.com/broken", alertCaptor.getValue().getContent());
}
@Test
void testUnparseableNumberFieldDoesNotAbortRule() throws InterruptedException {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
builder.setId(518679137103105L)
.setApp("fullsite")
.setMetrics("summary")
.setPriority(1)
.setCode(CollectRep.Code.SUCCESS);
CollectRep.Field url = CollectRep.Field.newBuilder().setName("url").setType(CommonConstants.TYPE_STRING).setLabel(true).build();
CollectRep.Field responseTime = CollectRep.Field.newBuilder().setName("responseTime").setType(CommonConstants.TYPE_NUMBER).build();
Map<String, String> meta = new HashMap<>();
meta.put(MetricDataConstants.INSTANCE_NAME, "site");
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
builder.addMetadataAll(meta);
builder.addAllFields(Lists.newArrayList(url, responseTime));
builder.addValueRow(CollectRep.ValueRow.newBuilder()
.addColumn("https://example.com/a").addColumn("").build());
CollectRep.MetricsData metricsData = builder.build();
AlertDefine guardedDefine = new AlertDefine();
guardedDefine.setId(2L);
guardedDefine.setName("slow-site");
guardedDefine.setExpr("equals(__app__,\"fullsite\") && exists(responseTime) && responseTime > 100");
guardedDefine.setTemplate("slow: ${url}");
guardedDefine.setTimes(1);
when(alertDefineService.getMetricsRealTimeAlertDefines()).thenReturn(Collections.singletonList(guardedDefine));
when(dataQueue.pollMetricsDataToAlerter()).thenReturn(metricsData).thenThrow(new InterruptedException());
metricsRealTimeAlertCalculator.startCalculate();
Thread.sleep(3000);
// unparseable number is defined as null: exists() short-circuits to false, no alarm and no abort
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any());
verify(alarmCacheManager, times(1)).removeFiring(any(), any());
}
}
@@ -69,6 +69,7 @@ public class AlertDefineExcelImExportServiceTest {
row.createCell(6).setCellValue(JsonUtil.toJson(Map.of("key", "value")));
row.createCell(7).setCellValue("template1");
row.createCell(8).setCellValue(true);
row.createCell(9).setCellValue("promql");
ByteArrayInputStream inputStream = new ByteArrayInputStream(toByteArray(initialWorkbook));
@@ -93,6 +94,7 @@ public class AlertDefineExcelImExportServiceTest {
assertEquals(Map.of("key", "value"), alertDefineDTO.getAnnotations());
assertEquals("template1", alertDefineDTO.getTemplate());
assertTrue(alertDefineDTO.getEnable());
assertEquals("promql", alertDefineDTO.getDatasource());
}
}
@@ -111,6 +113,7 @@ public class AlertDefineExcelImExportServiceTest {
alertDefineDTO.setAnnotations(Map.of("key", "value"));
alertDefineDTO.setTemplate("template1");
alertDefineDTO.setEnable(true);
alertDefineDTO.setDatasource("promql");
exportAlertDefineDTO.setAlertDefine(alertDefineDTO);
exportAlertDefineList.add(exportAlertDefineDTO);
@@ -129,6 +132,7 @@ public class AlertDefineExcelImExportServiceTest {
assertEquals("Annotations", headerRow.getCell(6).getStringCellValue());
assertEquals("Template", headerRow.getCell(7).getStringCellValue());
assertEquals("Enable", headerRow.getCell(8).getStringCellValue());
assertEquals("Datasource", headerRow.getCell(9).getStringCellValue());
Row dataRow = resultSheet.getRow(1);
assertEquals("app1", dataRow.getCell(0).getStringCellValue());
@@ -140,6 +144,7 @@ public class AlertDefineExcelImExportServiceTest {
assertEquals(JsonUtil.toJson(Map.of("key", "value")), dataRow.getCell(6).getStringCellValue());
assertEquals("template1", dataRow.getCell(7).getStringCellValue());
assertTrue(dataRow.getCell(8).getBooleanCellValue());
assertEquals("promql", dataRow.getCell(9).getStringCellValue());
}
}
}
@@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
@@ -29,8 +31,10 @@ import java.util.List;
import org.apache.hertzbeat.alert.dto.AlertDefineDTO;
import org.apache.hertzbeat.alert.dto.ExportAlertDefineDTO;
import org.apache.hertzbeat.alert.service.impl.AlertDefineJsonImExportServiceImpl;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.test.util.ReflectionTestUtils;
/**
* test case for {@link AlertDefineJsonImExportServiceImpl}
@@ -43,7 +47,7 @@ class AlertDefineJsonImExportServiceTest {
@SuppressWarnings("checkstyle:OperatorWrap")
private static final String JSON_DATA = "[{\"alertDefine\":{\"name\":\"App1\",\"type\":\"realtime\"," +
"\"expr\":\"Expr1\",\"period\":3000,\"times\":3," +
"\"enable\":true,\"template\":\"Template1\"}}]";
"\"enable\":true,\"template\":\"Template1\",\"datasource\":\"promql\"}}]";
private InputStream inputStream;
private List<ExportAlertDefineDTO> alertDefineList;
@@ -77,6 +81,7 @@ class AlertDefineJsonImExportServiceTest {
assertEquals(1, result.size());
assertEquals("App1", result.get(0).getAlertDefine().getName());
assertEquals("realtime", result.get(0).getAlertDefine().getType());
assertEquals("promql", result.get(0).getAlertDefine().getDatasource());
}
@Test
@@ -100,6 +105,29 @@ class AlertDefineJsonImExportServiceTest {
assertTrue(result.contains("realtime"));
}
@Test
void testExportKeepsDatasource() {
AlertDefineService alertDefineService = mock(AlertDefineService.class);
AlertDefine define = AlertDefine.builder()
.name("test")
.type("periodic_metric")
.expr("cpu_usage{instance=\"server1\"} > 80")
.datasource("promql")
.period(300)
.times(3)
.template("test")
.enable(true)
.build();
when(alertDefineService.getAlertDefine(1L)).thenReturn(define);
ReflectionTestUtils.setField(service, "alertDefineService", alertDefineService);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
service.exportConfig(outputStream, List.of(1L));
String result = outputStream.toString(StandardCharsets.UTF_8);
assertTrue(result.contains("promql"), "exported config should keep datasource, but got: " + result);
}
@Test
void testType() {
assertEquals("JSON", service.type());
@@ -64,6 +64,7 @@ class AlertDefineYamlImExportServiceTest {
times: 3
enable: true
template: Template1
datasource: promql
""";
private InputStream inputStream;
@@ -82,6 +83,7 @@ class AlertDefineYamlImExportServiceTest {
alertDefine.setExpr("Expr1");
alertDefine.setEnable(true);
alertDefine.setTemplate("Template1");
alertDefine.setDatasource("promql");
ExportAlertDefineDTO exportAlertDefine = new ExportAlertDefineDTO();
exportAlertDefine.setAlertDefine(alertDefine);
@@ -135,6 +137,7 @@ class AlertDefineYamlImExportServiceTest {
assertTrue(yamlOutput.contains("name: App1"));
assertTrue(yamlOutput.contains("type: realtime"));
assertTrue(yamlOutput.contains("expr: Expr1"));
assertTrue(yamlOutput.contains("datasource: promql"));
}
@Test
@@ -0,0 +1,180 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import java.time.Instant;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.service.impl.AlibabaCloudCmsExternAlertService;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Unit test for {@link AlibabaCloudCmsExternAlertService}.
*/
@ExtendWith(MockitoExtension.class)
class AlibabaCloudCmsExternAlertServiceTest {
private static final long EVENT_TIME = 1785300000123L;
@Mock
private AlarmCommonReduce alarmCommonReduce;
private AlibabaCloudCmsExternAlertService externAlertService;
@BeforeEach
void setUp() {
externAlertService = new AlibabaCloudCmsExternAlertService(alarmCommonReduce);
}
@Test
void shouldConvertTriggeredAlert() {
externAlertService.addExternAlert("""
{
"specversion": "1.0",
"id": "alert-event-1",
"type": "ALERT",
"subtype": "NORMAL_TRIGGER",
"time": "2026-07-29T06:00:00Z",
"timestamp": 1785300000123,
"subject": "ECS CPU usage is high",
"severity": "WARNING",
"status": "OCCURRED",
"userId": "123456",
"ruleId": "rule-1",
"workspace": "default-cms-123456-cn-hangzhou",
"traceId": "trace-1",
"alertMessage": "CPU usage exceeded 80%",
"alertEntityId": "ecs:i-123",
"resource": {
"entity": {
"domain": "ecs",
"entity_type": "instance",
"entity_id": "i-123",
"prop": {
"instanceName": "api-server"
}
},
"tags": {
"regionId": "cn-hangzhou",
"environment": "production"
}
},
"labels": {
"_cms_region": "cn-hangzhou",
"customNumber": 7
},
"annotations": {
"current_value": "92.5"
},
"data": {
"value": 92.5,
"threshold": 80,
"comparisonOperator": ">"
},
"alertEntityFields": {
"privateIp": "10.0.0.1"
},
"alertHistoryUrl": "https://cmsnext.console.aliyun.com/history",
"futureField": "ignored"
}
""");
SingleAlert alert = captureAlert();
assertEquals(CommonConstants.ALERT_STATUS_FIRING, alert.getStatus());
assertEquals(EVENT_TIME, alert.getStartAt());
assertEquals(EVENT_TIME, alert.getActiveAt());
assertNull(alert.getEndAt());
assertEquals("CPU usage exceeded 80%", alert.getContent());
assertEquals("alibabacloud-cms", alert.getLabels().get("__source__"));
assertEquals("ECS CPU usage is high", alert.getLabels().get("alertname"));
assertEquals(CommonConstants.ALERT_SEVERITY_WARNING, alert.getLabels().get("severity"));
assertEquals("instance", alert.getLabels().get("resourceType"));
assertEquals("i-123", alert.getLabels().get("resourceId"));
assertEquals("7", alert.getLabels().get("customNumber"));
assertEquals("api-server", alert.getAnnotations().get("instanceName"));
assertEquals("92.5", alert.getAnnotations().get("value"));
assertEquals("80", alert.getAnnotations().get("threshold"));
assertEquals("10.0.0.1", alert.getAnnotations().get("privateIp"));
}
@Test
void shouldConvertResolvedAlertAndIsoTime() {
externAlertService.addExternAlert("""
{
"subtype": "NORMAL_RESOLVE",
"time": "2026-07-29T06:00:00Z",
"subject": "ECS CPU usage is high",
"severity": "CRITICAL",
"status": "RESOLVED",
"labels": {
"instanceId": "i-123"
}
}
""");
SingleAlert alert = captureAlert();
long expectedTime = Instant.parse("2026-07-29T06:00:00Z").toEpochMilli();
assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alert.getStatus());
assertEquals(expectedTime, alert.getStartAt());
assertNull(alert.getActiveAt());
assertEquals(expectedTime, alert.getEndAt());
assertEquals("ECS CPU usage is high", alert.getContent());
assertEquals(CommonConstants.ALERT_SEVERITY_CRITICAL, alert.getLabels().get("severity"));
}
@Test
void shouldTreatRecoveredStatusAsResolved() {
externAlertService.addExternAlert("""
{
"timestamp": 1785300000123,
"subject": "Recovered alert",
"status": "RECOVERED"
}
""");
SingleAlert alert = captureAlert();
assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alert.getStatus());
assertEquals(EVENT_TIME, alert.getEndAt());
}
@Test
void shouldIgnoreInvalidPayload() {
externAlertService.addExternAlert("invalid json");
externAlertService.addExternAlert("{\"subject\":\"missing status\"}");
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any(SingleAlert.class));
assertEquals("alibabacloud-cms", externAlertService.supportSource());
}
private SingleAlert captureAlert() {
ArgumentCaptor<SingleAlert> captor = ArgumentCaptor.forClass(SingleAlert.class);
verify(alarmCommonReduce).reduceAndSendAlarm(captor.capture());
return captor.getValue();
}
}
@@ -151,8 +151,8 @@
<!--Bouncy Castle-->
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<version>1.68</version>
<artifactId>bcpkix-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
@@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.common;
import java.util.Collections;
import java.util.List;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.springframework.util.StringUtils;
/**
* Shared one-row response handling for command-based collectors.
*/
public final class OneRowResponseSupport {
/**
* Parse type where each output line maps to one alias field of a single result row.
*/
public static final String PARSE_TYPE_ONE_ROW = "oneRow";
private OneRowResponseSupport() {
}
/**
* Treat blank stdout without an error signal (no stderr, exit status present and &lt;= 1,
* grep-style no match) as valid empty one-row data: append a row of null placeholders so the
* metric stays visible and alertable.
*
* @return true if handled as empty success, false if the caller should report a failure
*/
public static boolean tryAppendEmptyOneRow(String parseType, String stdErr, Integer exitStatus,
List<String> aliasFields, CollectRep.MetricsData.Builder builder,
Long responseTime) {
if (PARSE_TYPE_ONE_ROW.equals(parseType)
&& !StringUtils.hasText(stdErr)
&& exitStatus != null && exitStatus <= 1) {
appendEmptyValues(aliasFields, builder, responseTime);
return true;
}
return false;
}
/**
* Build the failure message for a command that produced no usable stdout: prefer the captured
* stderr, then a non-trivial exit status, otherwise the generic null-data message.
*/
public static String buildBlankFailureMessage(String stdErr, Integer exitStatus,
String exitCodePrefix, String nullMessage) {
if (StringUtils.hasText(stdErr)) {
return stdErr.trim();
}
if (exitStatus != null && exitStatus > 1) {
return exitCodePrefix + exitStatus;
}
return nullMessage;
}
/**
* Map each output line to one alias field of a single row; missing trailing lines become
* NULL_VALUE columns so a partial result keeps its values and the gap stays alertable.
*/
public static void appendResponseValues(String result, List<String> aliasFields,
CollectRep.MetricsData.Builder builder, Long responseTime) {
List<String> safeAliasFields = aliasFields == null ? Collections.emptyList() : aliasFields;
String[] lines = result.split("\n");
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
int aliasIndex = 0;
int lineIndex = 0;
while (aliasIndex < safeAliasFields.size()) {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(safeAliasFields.get(aliasIndex))) {
valueRowBuilder.addColumn(responseTime.toString());
} else {
if (lineIndex < lines.length) {
valueRowBuilder.addColumn(lines[lineIndex].trim());
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
lineIndex++;
}
aliasIndex++;
}
builder.addValueRow(valueRowBuilder.build());
}
public static void appendEmptyValues(List<String> aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) {
List<String> safeAliasFields = aliasFields == null ? Collections.emptyList() : aliasFields;
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String aliasField : safeAliasFields) {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasField)) {
valueRowBuilder.addColumn(responseTime.toString());
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
}
builder.addValueRow(valueRowBuilder.build());
}
}
@@ -704,13 +704,11 @@ public class HttpCollectImpl extends AbstractCollect {
valueRowBuilder.addColumn(String.valueOf(value));
} else {
if (alias.startsWith("$.")) {
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1));
if (subResults != null && subResults.size() > i) {
Object resultValue = subResults.get(i);
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
// per-row evaluation, a global "parseScript + alias" query would misalign rows missing the path
List<Object> aliasValues = JsonPathParser.parseRowWithJsonPath(objectValue, alias);
// a wildcard alias matching multiple values is kept whole and rendered as "[v1, v2]"
Object resultValue = aliasValues.size() == 1 ? aliasValues.get(0) : (aliasValues.isEmpty() ? null : aliasValues);
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
} else {
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
}
@@ -60,16 +60,37 @@ public class OnlineParser {
}
public static Map<String, MetricFamily> parseMetrics(InputStream inputStream) throws IOException {
Map<String, MetricFamily> metricFamilyMap = new ConcurrentHashMap<>(10);
return parseMetrics(inputStream, Integer.MAX_VALUE);
}
/**
* Parses at most {@code maxSamples} samples from the supplied stream.
*
* @param inputStream The Prometheus text stream
* @param maxSamples The maximum number of samples to materialize
* @return The parsed metric families, or {@code null} when the text format is invalid
* @throws IOException When the stream cannot be read
* @throws SampleLimitExceededException When another sample follows the configured limit
*/
public static Map<String, MetricFamily> parseMetrics(InputStream inputStream, int maxSamples) throws IOException {
if (maxSamples < 0) {
throw new IllegalArgumentException("maxSamples must not be negative");
}
final Map<String, MetricFamily> metricFamilyMap = new ConcurrentHashMap<>(10);
int sampleCount = 0;
try {
int i = getChar(inputStream);
while (i != -1) {
if (i == '#' || i == '\n') {
skipToLineEnd(inputStream).maybeEol().maybeEof().noElse();
} else {
StringBuilder stringBuilder = new StringBuilder();
if (sampleCount >= maxSamples) {
throw new SampleLimitExceededException(maxSamples);
}
final StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append((char) i);
parseMetric(inputStream, metricFamilyMap, stringBuilder);
sampleCount++;
}
i = getChar(inputStream);
// To address the `\n\r` scenario, it is necessary to skip
@@ -84,6 +105,16 @@ public class OnlineParser {
return metricFamilyMap;
}
/**
* Signals that parsing stopped before materializing a sample beyond the configured limit.
*/
public static final class SampleLimitExceededException extends IOException {
public SampleLimitExceededException(int limit) {
super("prometheus payload exceeds the " + limit + " sample limit");
}
}
/**
* Parses Prometheus metrics from the given {@link InputStream}, but only for the specified metric name.
* <p>
@@ -30,6 +30,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.OneRowResponseSupport;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
@@ -52,7 +53,6 @@ public class ScriptCollectImpl extends AbstractCollect {
private static final String BASH_C = "-c";
private static final String POWERSHELL_C = "-Command";
private static final String POWERSHELL_FILE = "-File";
private static final String PARSE_TYPE_ONE_ROW = "oneRow";
private static final String PARSE_TYPE_MULTI_ROW = "multiRow";
private static final String PARSE_TYPE_NETCAT = "netcat";
private static final String PARSE_TYPE_LOG = "log";
@@ -113,25 +113,48 @@ public class ScriptCollectImpl extends AbstractCollect {
try {
Process process = processBuilder.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName(scriptProtocol.getCharset())));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
if (StringUtils.hasText(line)) {
response.append(line).append("\n");
BufferedReader errorReader = new BufferedReader(
new InputStreamReader(process.getErrorStream(), Charset.forName(scriptProtocol.getCharset())));
// drain stderr on its own thread: a full stderr pipe would deadlock the stdout read;
// StringBuffer because the drainer may still be writing when the buffer is read
StringBuffer errorBuffer = new StringBuffer();
Thread errorDrainer = new Thread(() -> {
try {
String errorLine;
while ((errorLine = errorReader.readLine()) != null) {
if (StringUtils.hasText(errorLine)) {
errorBuffer.append(errorLine).append("\n");
}
}
} catch (IOException e) {
log.warn("read script error stream failed: {}", e.getMessage());
}
}
process.waitFor();
});
errorDrainer.setDaemon(true);
errorDrainer.start();
String result = readResponse(reader);
int exitCode = process.waitFor();
// bounded: a lingering grandchild can keep the stderr pipe open
errorDrainer.join(1000);
Long responseTime = System.currentTimeMillis() - startTime;
String result = String.valueOf(response);
String errorResult = errorBuffer.toString();
if (!StringUtils.hasText(result)) {
if (OneRowResponseSupport.tryAppendEmptyOneRow(scriptProtocol.getParseType(), errorResult,
exitCode, metrics.getAliasFields(), builder, responseTime)) {
return;
}
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("Script response data is null");
builder.setMsg(OneRowResponseSupport.buildBlankFailureMessage(errorResult, exitCode,
"Script exited with code: ", "Script response data is null"));
return;
}
if (StringUtils.hasText(errorResult)) {
log.warn("script command succeeded but wrote to stderr: {}", errorResult.trim());
}
switch (scriptProtocol.getParseType()) {
case PARSE_TYPE_LOG -> parseResponseDataByLog(result, metrics.getAliasFields(), builder, responseTime);
case PARSE_TYPE_NETCAT -> parseResponseDataByNetcat(result, metrics.getAliasFields(), builder, responseTime);
case PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
case OneRowResponseSupport.PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
case PARSE_TYPE_MULTI_ROW -> parseResponseDataByMulti(result, metrics.getAliasFields(), builder, responseTime);
default -> {
builder.setCode(CollectRep.Code.FAIL);
@@ -207,28 +230,7 @@ public class ScriptCollectImpl extends AbstractCollect {
}
private void parseResponseDataByOne(String result, List<String> aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) {
String[] lines = result.split("\n");
if (lines.length + 1 < aliasFields.size()) {
log.error("ssh response data not enough: {}", result);
return;
}
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
int aliasIndex = 0;
int lineIndex = 0;
while (aliasIndex < aliasFields.size()) {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasFields.get(aliasIndex))) {
valueRowBuilder.addColumn(responseTime.toString());
} else {
if (lineIndex < lines.length) {
valueRowBuilder.addColumn(lines[lineIndex].trim());
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
lineIndex++;
}
aliasIndex++;
}
builder.addValueRow(valueRowBuilder.build());
OneRowResponseSupport.appendResponseValues(result, aliasFields, builder, responseTime);
}
private void parseResponseDataByMulti(String result, List<String> aliasFields,
@@ -261,4 +263,15 @@ public class ScriptCollectImpl extends AbstractCollect {
builder.addValueRow(valueRowBuilder.build());
}
}
private String readResponse(BufferedReader reader) throws IOException {
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
if (StringUtils.hasText(line)) {
response.append(line).append("\n");
}
}
return response.toString();
}
}
@@ -21,6 +21,8 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ConnectException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.net.SocketTimeoutException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
@@ -33,6 +35,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollect;
import org.apache.hertzbeat.collector.collect.common.OneRowResponseSupport;
import org.apache.hertzbeat.collector.collect.common.ssh.CommonSshBlacklist;
import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
@@ -49,7 +52,6 @@ import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.common.SshException;
import org.apache.sshd.common.channel.exception.SshChannelOpenException;
import org.apache.sshd.common.future.CloseFuture;
import org.apache.sshd.common.util.io.output.NoCloseOutputStream;
import org.springframework.util.StringUtils;
/**
@@ -58,7 +60,6 @@ import org.springframework.util.StringUtils;
@Slf4j
public class SshCollectImpl extends AbstractCollect {
private static final String PARSE_TYPE_ONE_ROW = "oneRow";
private static final String PARSE_TYPE_MULTI_ROW = "multiRow";
private static final String PARSE_TYPE_NETCAT = "netcat";
private static final String PARSE_TYPE_LOG = "log";
@@ -93,8 +94,9 @@ public class SshCollectImpl extends AbstractCollect {
}
channel = clientSession.createExecChannel(sshProtocol.getScript());
ByteArrayOutputStream response = new ByteArrayOutputStream();
ByteArrayOutputStream errorResponse = new ByteArrayOutputStream();
channel.setOut(response);
channel.setErr(new NoCloseOutputStream(System.err));
channel.setErr(errorResponse);
channel.open().verify(timeout);
List<ClientChannelEvent> list = new ArrayList<>();
list.add(ClientChannelEvent.CLOSED);
@@ -107,16 +109,28 @@ public class SshCollectImpl extends AbstractCollect {
throw new SocketTimeoutException("Failed to retrieve command result in time: " + sshProtocol.getScript());
}
Long responseTime = System.currentTimeMillis() - startTime;
String result = response.toString();
Charset charset = StringUtils.hasText(sshProtocol.getCharset())
? Charset.forName(sshProtocol.getCharset()) : StandardCharsets.UTF_8;
String result = response.toString(charset);
String errorResult = errorResponse.toString(charset);
Integer exitStatus = channel.getExitStatus();
if (!StringUtils.hasText(result)) {
if (OneRowResponseSupport.tryAppendEmptyOneRow(sshProtocol.getParseType(), errorResult,
exitStatus, metrics.getAliasFields(), builder, responseTime)) {
return;
}
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("ssh shell response data is null");
builder.setMsg(OneRowResponseSupport.buildBlankFailureMessage(errorResult, exitStatus,
"ssh command exited with code: ", "ssh shell response data is null"));
return;
}
if (StringUtils.hasText(errorResult)) {
log.warn("ssh command succeeded but wrote to stderr: {}", errorResult.trim());
}
switch (sshProtocol.getParseType()) {
case PARSE_TYPE_LOG -> parseResponseDataByLog(result, metrics.getAliasFields(), builder, responseTime);
case PARSE_TYPE_NETCAT -> parseResponseDataByNetcat(result, metrics.getAliasFields(), builder, responseTime);
case PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
case OneRowResponseSupport.PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
case PARSE_TYPE_MULTI_ROW -> parseResponseDataByMulti(result, metrics.getAliasFields(), builder, responseTime);
default -> {
builder.setCode(CollectRep.Code.FAIL);
@@ -244,28 +258,7 @@ public class SshCollectImpl extends AbstractCollect {
}
private void parseResponseDataByOne(String result, List<String> aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) {
String[] lines = result.split("\n");
if (lines.length + 1 < aliasFields.size()) {
log.error("ssh response data not enough: {}", result);
return;
}
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
int aliasIndex = 0;
int lineIndex = 0;
while (aliasIndex < aliasFields.size()) {
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasFields.get(aliasIndex))) {
valueRowBuilder.addColumn(responseTime.toString());
} else {
if (lineIndex < lines.length) {
valueRowBuilder.addColumn(lines[lineIndex].trim());
} else {
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
}
lineIndex++;
}
aliasIndex++;
}
builder.addValueRow(valueRowBuilder.build());
OneRowResponseSupport.appendResponseValues(result, aliasFields, builder, responseTime);
}
private void parseResponseDataByMulti(String result, List<String> aliasFields,
@@ -67,14 +67,20 @@ public class TelnetCollectImpl extends AbstractCollect {
long responseTime = System.currentTimeMillis() - startTime;
List<String> aliasFields = metrics.getAliasFields();
String app = builder.getApp();
Map<String, String> resultMap = execCmdAndParseResult(telnetClient, telnet.getCmd(), app);
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
if (resultMap.size() < aliasFields.size()) {
log.error("telnet response data not enough: {}", resultMap);
CmdResult cmdResult = execCmdAndParseResult(telnetClient, telnet.getCmd(), app);
Map<String, String> resultMap = cmdResult.values();
boolean expectsCmdMetrics = StringUtils.isNotBlank(telnet.getCmd())
&& aliasFields.stream().anyMatch(field -> !CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(field));
boolean hasExpectedMetric = aliasFields.stream().anyMatch(resultMap::containsKey);
if (expectsCmdMetrics && !hasExpectedMetric) {
// e.g. zookeeper refusing a 4lw command not in its 4lw.commands.whitelist
String reply = sanitizeReply(cmdResult.rawResponse());
log.warn("telnet cmd [{}] returned no expected metrics: {}", telnet.getCmd(), reply);
builder.setCode(CollectRep.Code.FAIL);
builder.setMsg("The cmd execution results do not match the expected number of metrics.");
builder.setMsg("Cmd [" + telnet.getCmd() + "] returned no expected metrics. Response: " + reply);
return;
}
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : aliasFields) {
String fieldValue = resultMap.get(field);
@@ -118,30 +124,36 @@ public class TelnetCollectImpl extends AbstractCollect {
return DispatchConstants.PROTOCOL_TELNET;
}
private static Map<String, String> execCmdAndParseResult(TelnetClient telnetClient, String cmd, String app) throws IOException {
record CmdResult(Map<String, String> values, String rawResponse) {
}
private static String sanitizeReply(String raw) {
return StringUtils.abbreviate(raw.trim().replaceAll("[\\p{Cntrl}]+", " "), 300);
}
private static CmdResult execCmdAndParseResult(TelnetClient telnetClient, String cmd, String app) throws IOException {
if (cmd == null || StringUtils.isEmpty(cmd.trim())) {
return new HashMap<>(16);
return new CmdResult(new HashMap<>(16), "");
}
OutputStream outputStream = telnetClient.getOutputStream();
outputStream.write(cmd.getBytes(StandardCharsets.UTF_8));
outputStream.flush();
String result = new String(telnetClient.getInputStream().readAllBytes());
String[] lines = result.split("\n");
if (CollectorConstants.ZOOKEEPER_APP.equals(app) && CollectorConstants.ZOOKEEPER_ENVI_HEAD.equals(lines[0])) {
if (lines.length > 0 && CollectorConstants.ZOOKEEPER_APP.equals(app)
&& CollectorConstants.ZOOKEEPER_ENVI_HEAD.equals(lines[0])) {
lines = Arrays.stream(lines)
.skip(1)
.toArray(String[]::new);
}
boolean contains = lines[0].contains("=");
return Arrays.stream(lines)
.map(item -> {
if (contains) {
return item.split("=");
} else {
return item.split("\t");
}
})
if (lines.length == 0) {
return new CmdResult(new HashMap<>(16), result);
}
String separator = lines[0].contains("=") ? "=" : "\t";
Map<String, String> values = Arrays.stream(lines)
.map(item -> item.split(separator, 2))
.filter(item -> item.length == 2)
.collect(Collectors.toMap(x -> x[0], x -> x[1]));
.collect(Collectors.toMap(x -> x[0], x -> x[1], (first, second) -> first, HashMap::new));
return new CmdResult(values, result);
}
}
@@ -0,0 +1,123 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.common;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.apache.hertzbeat.collector.constants.CollectorConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.junit.jupiter.api.Test;
class OneRowResponseSupportTest {
@Test
void appendResponseValuesShouldMapColumnsInOrder() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
OneRowResponseSupport.appendResponseValues(
"pod-a\n5\n", List.of("pod", "restart", CollectorConstants.RESPONSE_TIME), builder, 18L);
assertEquals(1, builder.getValuesCount());
assertEquals("pod-a", builder.getValues(0).getColumns(0));
assertEquals("5", builder.getValues(0).getColumns(1));
assertEquals("18", builder.getValues(0).getColumns(2));
}
@Test
void appendResponseValuesShouldPadMissingTrailingLines() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
OneRowResponseSupport.appendResponseValues(
"52\n35.8033\n5%",
List.of("cpu", "memory", "disk", "nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 18L);
assertEquals(1, builder.getValuesCount());
assertEquals("52", builder.getValues(0).getColumns(0));
assertEquals("35.8033", builder.getValues(0).getColumns(1));
assertEquals("5%", builder.getValues(0).getColumns(2));
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
assertEquals("18", builder.getValues(0).getColumns(4));
}
@Test
void appendEmptyValuesShouldFillNullPlaceholders() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
OneRowResponseSupport.appendEmptyValues(
List.of("nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 12L);
assertEquals(1, builder.getValuesCount());
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
assertEquals("12", builder.getValues(0).getColumns(1));
}
@Test
void tryAppendEmptyOneRowShouldAcceptGrepNoMatchExitOne() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
// grep with no match exits 1 and writes nothing: treat as valid empty data, not a failure
boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow(
OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "", 1,
List.of("nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 9L);
assertTrue(handled);
assertEquals(1, builder.getValuesCount());
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
}
@Test
void tryAppendEmptyOneRowShouldRejectNullExitStatus() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
// an absent exit status (e.g. dropped ssh channel) must be treated as a failure
boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow(
OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "", null,
List.of("nfs_mount"), builder, 9L);
assertFalse(handled);
assertEquals(0, builder.getValuesCount());
}
@Test
void tryAppendEmptyOneRowShouldRejectNonEmptyStderr() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow(
OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "permission denied", 1,
List.of("nfs_mount"), builder, 9L);
assertFalse(handled);
assertEquals(0, builder.getValuesCount());
}
@Test
void buildBlankFailureMessageShouldPreferStderrThenExitCode() {
assertEquals("permission denied", OneRowResponseSupport.buildBlankFailureMessage(
"permission denied\n", 2, "cmd exited with code: ", "null data"));
assertEquals("cmd exited with code: 2", OneRowResponseSupport.buildBlankFailureMessage(
"", 2, "cmd exited with code: ", "null data"));
assertEquals("null data", OneRowResponseSupport.buildBlankFailureMessage(
"", 1, "cmd exited with code: ", "null data"));
assertEquals("null data", OneRowResponseSupport.buildBlankFailureMessage(
"", null, "cmd exited with code: ", "null data"));
}
}
@@ -20,6 +20,7 @@ package org.apache.hertzbeat.collector.collect.http;
import com.google.common.collect.Lists;
import com.sun.net.httpserver.HttpServer;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
@@ -383,6 +384,48 @@ class HttpCollectImplTest {
assertEquals("0.268751364291017", firstRow.getColumns(0));
}
@Test
void parseResponseByJsonPathKeepsRowAlignmentWhenAliasPathMissing() throws Exception {
String jsonResponse = "{\"items\": ["
+ "{\"metadata\": {\"name\": \"pod-a\"}, \"status\": {\"phase\": \"Running\","
+ " \"containerStatuses\": [{\"name\": \"c1\", \"ready\": true, \"restartCount\": 5}]}},"
+ "{\"metadata\": {\"name\": \"pod-b-pending\"}, \"status\": {\"phase\": \"Pending\"}},"
+ "{\"metadata\": {\"name\": \"pod-c\"}, \"status\": {\"phase\": \"Running\","
+ " \"containerStatuses\": [{\"name\": \"c3\", \"ready\": true, \"restartCount\": 2}]}}"
+ "]}";
HttpProtocol http = HttpProtocol.builder()
.parseType(DispatchConstants.PARSE_JSON_PATH)
.parseScript("$.items.*")
.build();
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
@Override
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
capturedRows.add(valueRow);
return super.addValueRow(valueRow);
}
};
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
"parseResponseByJsonPath",
String.class,
List.class,
HttpProtocol.class,
CollectRep.MetricsData.Builder.class,
Long.class);
parseMethod.setAccessible(true);
parseMethod.invoke(httpCollectImpl, jsonResponse,
Lists.newArrayList("$.metadata.name", "$.status.containerStatuses[0].restartCount"), http, builder, 100L);
assertEquals(3, capturedRows.size());
assertEquals("pod-a", capturedRows.get(0).getColumns(0));
assertEquals("5", capturedRows.get(0).getColumns(1));
assertEquals("pod-b-pending", capturedRows.get(1).getColumns(0));
assertEquals(CommonConstants.NULL_VALUE, capturedRows.get(1).getColumns(1));
assertEquals("pod-c", capturedRows.get(2).getColumns(0));
assertEquals("2", capturedRows.get(2).getColumns(1));
}
@Test
void testParsePromQlLabelValue() throws Exception {
// Create Prometheus format test data
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.mqtt;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.io.StringWriter;
import java.math.BigInteger;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.cert.X509Certificate;
import java.util.Date;
import org.apache.hertzbeat.common.entity.job.protocol.MqttProtocol;
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
import org.bouncycastle.openssl.jcajce.JcaPKCS8Generator;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
class MqttSslFactoryTest {
private static String certPem;
private static String pkcs1KeyPem;
private static String pkcs8KeyPem;
@BeforeAll
static void generateCertAndKeys() throws Exception {
KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
generator.initialize(2048);
KeyPair keyPair = generator.generateKeyPair();
X500Name subject = new X500Name("CN=hb-3540-mqtt");
JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
subject, BigInteger.ONE,
new Date(System.currentTimeMillis() - 60_000),
new Date(System.currentTimeMillis() + 3_600_000),
subject, keyPair.getPublic());
X509Certificate cert = new JcaX509CertificateConverter()
.getCertificate(certBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate())));
certPem = writePem(cert);
pkcs1KeyPem = writePem(keyPair.getPrivate());
pkcs8KeyPem = writePem(new JcaPKCS8Generator(keyPair.getPrivate(), null));
}
@Test
void parsesPkcs1ClientKey() {
assertNotNull(MqttSslFactory.getMslSocketFactory(mqttProtocol(pkcs1KeyPem), true));
}
@Test
void parsesPkcs8ClientKey() {
assertNotNull(MqttSslFactory.getMslSocketFactory(mqttProtocol(pkcs8KeyPem), true));
}
private static MqttProtocol mqttProtocol(String clientKey) {
return MqttProtocol.builder()
.tlsVersion("TLSv1.2")
.clientCert(certPem)
.clientKey(clientKey)
.build();
}
private static String writePem(Object object) throws Exception {
StringWriter out = new StringWriter();
try (JcaPEMWriter writer = new JcaPEMWriter(out)) {
writer.writeObject(object);
}
return out.toString();
}
}
@@ -30,6 +30,8 @@ import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
class OnlineParserTest {
@@ -459,4 +461,27 @@ class OnlineParserTest {
assertEquals("run_as", metricFamily.getMetricList().get(0).getLabels().get(3).getName());
assertEquals("NT AUTHORITY\nLocalService", metricFamily.getMetricList().get(0).getLabels().get(3).getValue());
}
@Test
void testParseMetricsStopsBeforeSampleBeyondLimit() {
final String metrics = "metric_a 1\nmetric_b 2\nmetric_c 3\nmetric_d 4\n";
final ByteArrayInputStream inputStream =
new ByteArrayInputStream(metrics.getBytes(StandardCharsets.UTF_8));
assertThrows(OnlineParser.SampleLimitExceededException.class,
() -> OnlineParser.parseMetrics(inputStream, 2));
assertTrue(inputStream.available() > 0, "samples after the limit should remain unread");
}
@Test
void testParseMetricsAllowsExactlyTheSampleLimit() throws Exception {
final String metrics = "metric_a 1\nmetric_b 2\n";
final InputStream inputStream = new ByteArrayInputStream(metrics.getBytes(StandardCharsets.UTF_8));
final Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream, 2);
assertNotNull(metricFamilyMap);
assertEquals(2, metricFamilyMap.size());
}
}
@@ -19,9 +19,12 @@ package org.apache.hertzbeat.collector.collect.script;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.List;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.ScriptProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
@@ -138,6 +141,85 @@ public class ScriptCollectImplTest {
scriptCollect.collect(builder, metrics);
assertEquals(CollectRep.Code.FAIL, builder.getCode());
});
// empty stdout without stderr should be treated as empty one-row data
assertDoesNotThrow(() -> {
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
.charset("utf-8")
.parseType("oneRow")
.scriptTool("bash")
.scriptCommand("grep -o 'centos-hermitlv' /dev/null")
.build();
Metrics metrics = new Metrics();
metrics.setScript(scriptProtocol);
metrics.setAliasFields(List.of("nfs_mount"));
builder = CollectRep.MetricsData.newBuilder();
scriptCollect.collect(builder, metrics);
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
assertEquals(1, builder.getValuesCount());
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
});
// partial output missing more than one trailing field: the old length check
// (lines + 1 < aliases) dropped the whole row here, losing the collected values
assertDoesNotThrow(() -> {
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
.charset("utf-8")
.parseType("oneRow")
.scriptTool("bash")
.scriptCommand("echo 52; echo 35.8033; grep -o 'centos-hermitlv' /dev/null")
.build();
Metrics metrics = new Metrics();
metrics.setScript(scriptProtocol);
metrics.setAliasFields(List.of("cpu", "memory", "disk", "nfs_mount"));
builder = CollectRep.MetricsData.newBuilder();
scriptCollect.collect(builder, metrics);
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
assertEquals(1, builder.getValuesCount());
assertEquals("52", builder.getValues(0).getColumns(0));
assertEquals("35.8033", builder.getValues(0).getColumns(1));
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(2));
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
});
// a command that silently exits 1 with no output is indistinguishable from a
// grep no-match, so it is deliberately accepted as an empty success
assertDoesNotThrow(() -> {
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
.charset("utf-8")
.parseType("oneRow")
.scriptTool("bash")
.scriptCommand("exit 1")
.build();
Metrics metrics = new Metrics();
metrics.setScript(scriptProtocol);
metrics.setAliasFields(List.of("nfs_mount"));
builder = CollectRep.MetricsData.newBuilder();
scriptCollect.collect(builder, metrics);
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
});
// non-empty exit code without stderr should still fail when it is not the grep-style no-match case
assertDoesNotThrow(() -> {
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
.charset("utf-8")
.parseType("oneRow")
.scriptTool("bash")
.scriptCommand("exit 2")
.build();
Metrics metrics = new Metrics();
metrics.setScript(scriptProtocol);
metrics.setAliasFields(List.of("nfs_mount"));
builder = CollectRep.MetricsData.newBuilder();
scriptCollect.collect(builder, metrics);
assertEquals(CollectRep.Code.FAIL, builder.getCode());
assertTrue(builder.getMsg().contains("code: 2"));
});
}
@Test
@@ -22,6 +22,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
@@ -30,14 +35,21 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.SshProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.sshd.client.channel.ChannelExec;
import org.apache.sshd.client.channel.ClientChannel;
import org.apache.sshd.client.channel.ClientChannelEvent;
import org.apache.sshd.client.future.OpenFuture;
import org.apache.sshd.client.session.ClientSession;
import org.apache.sshd.common.SshException;
@@ -229,6 +241,130 @@ class SshCollectImplTest {
verify(clientSession).close();
}
@Test
void collectPadsPartialOneRowOutput() throws Exception {
ChannelExec channel = oneRowChannel("52\n35.8033\n5%", "", 0);
Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build();
metrics.setAliasFields(List.of("cpu", "memory", "disk", "nfs_mount"));
ClientSession clientSession = channelSession(channel);
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
.thenReturn(clientSession);
sshCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
assertEquals(1, builder.getValuesCount());
assertEquals("52", builder.getValues(0).getColumns(0));
assertEquals("35.8033", builder.getValues(0).getColumns(1));
assertEquals("5%", builder.getValues(0).getColumns(2));
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
}
@Test
void collectTreatsSilentEmptyOneRowOutputAsEmptyRow() throws Exception {
ChannelExec channel = oneRowChannel("", "", 1);
Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build();
metrics.setAliasFields(List.of("nfs_mount"));
ClientSession clientSession = channelSession(channel);
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
.thenReturn(clientSession);
sshCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
}
@Test
void collectFailsOnEmptyOutputWithStderr() throws Exception {
ChannelExec channel = oneRowChannel("", "boom: permission denied", 1);
Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build();
metrics.setAliasFields(List.of("nfs_mount"));
ClientSession clientSession = channelSession(channel);
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
.thenReturn(clientSession);
sshCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.FAIL, builder.getCode());
assertEquals("boom: permission denied", builder.getMsg());
}
@Test
void collectDecodesOutputWithConfiguredCharset() throws Exception {
ChannelExec channel = oneRowChannel("挂载正常".getBytes(Charset.forName("GBK")), new byte[0], 0);
SshProtocol protocol = oneRowProtocol();
protocol.setCharset("GBK");
Metrics metrics = Metrics.builder().ssh(protocol).build();
metrics.setAliasFields(List.of("nfs_mount"));
ClientSession clientSession = channelSession(channel);
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
.thenReturn(clientSession);
sshCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
assertEquals("挂载正常", builder.getValues(0).getColumns(0));
}
private SshProtocol oneRowProtocol() {
return SshProtocol.builder()
.host("target.example.com")
.port("22")
.username("root")
.password("password")
.timeout("1000")
.reuseConnection("true")
.useProxy("false")
.script("echo ok")
.parseType("oneRow")
.build();
}
private ClientSession channelSession(ChannelExec channel) throws IOException {
ClientSession clientSession = mock(ClientSession.class);
when(clientSession.createExecChannel("echo ok")).thenReturn(channel);
return clientSession;
}
private ChannelExec oneRowChannel(String stdout, String stderr, int exitStatus) throws IOException {
return oneRowChannel(stdout.getBytes(StandardCharsets.UTF_8), stderr.getBytes(StandardCharsets.UTF_8), exitStatus);
}
private ChannelExec oneRowChannel(byte[] stdout, byte[] stderr, int exitStatus) throws IOException {
ChannelExec channel = mock(ChannelExec.class);
OpenFuture openFuture = mock(OpenFuture.class);
CloseFuture closeFuture = mock(CloseFuture.class);
AtomicReference<OutputStream> out = new AtomicReference<>();
AtomicReference<OutputStream> err = new AtomicReference<>();
doAnswer(inv -> {
out.set(inv.getArgument(0));
return null;
}).when(channel).setOut(any());
doAnswer(inv -> {
err.set(inv.getArgument(0));
return null;
}).when(channel).setErr(any());
when(channel.open()).thenReturn(openFuture);
when(channel.waitFor(any(), anyLong())).thenAnswer(inv -> {
out.get().write(stdout);
err.get().write(stderr);
return Set.of(ClientChannelEvent.CLOSED);
});
when(channel.getExitStatus()).thenReturn(exitStatus);
when(channel.close(false)).thenReturn(closeFuture);
when(closeFuture.await(anyLong())).thenReturn(true);
return channel;
}
private SshProtocol protocol(int timeout) {
return SshProtocol.builder()
.host("target.example.com")
@@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
@@ -30,6 +31,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.net.telnet.TelnetClient;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.TelnetProtocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
@@ -155,6 +157,114 @@ class TelnetCollectImplTest {
mocked.close();
}
@Test
void testCollectPadsMissingMetrics() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
Metrics metrics = telnetMetrics("mntr", List.of("responseTime", "a", "b", "c"));
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("a=1")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(1, builder.getValuesCount());
assertEquals("1", builder.getValues(0).getColumns(1));
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(2));
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
}
@Test
void testCollectFailsWithRawReplyWhenNothingParsed() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
Metrics metrics = telnetMetrics("conf", List.of("responseTime", "a"));
try (MockedConstruction<TelnetClient> mocked =
mockTelnetReply("conf is not executed because it is not in the whitelist.")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.FAIL, builder.getCode());
assertTrue(builder.getMsg().contains("not in the whitelist"));
}
@Test
void testCollectKeepsFirstOnDuplicateKeys() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
Metrics metrics = telnetMetrics("mntr", List.of("a", "b"));
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("a=1\na=2\nb=3")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(1, builder.getValuesCount());
assertEquals("1", builder.getValues(0).getColumns(0));
assertEquals("3", builder.getValues(0).getColumns(1));
}
@Test
void testCollectFailsWhenReplyHasOnlyUnrelatedPairs() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
Metrics metrics = telnetMetrics("mntr", List.of("responseTime", "a"));
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("error=conf disabled")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.FAIL, builder.getCode());
}
@Test
void testCollectSurvivesHeaderOnlyEnviReply() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder().setApp("zookeeper");
Metrics metrics = telnetMetrics("envi", List.of("responseTime", "a"));
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("Environment:")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.FAIL, builder.getCode());
}
@Test
void testCollectFailsCleanlyOnNewlineOnlyReply() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder().setApp("zookeeper");
Metrics metrics = telnetMetrics("conf", List.of("responseTime", "a"));
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("\n")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(CollectRep.Code.FAIL, builder.getCode());
assertTrue(builder.getMsg().contains("returned no expected metrics"));
}
@Test
void testCollectKeepsValueContainingSeparator() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
Metrics metrics = telnetMetrics("conf", List.of("dataDir", "secureClientPort"));
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("dataDir=/data/zk=a\nsecureClientPort=")) {
telnetCollect.collect(builder, metrics);
}
assertEquals(1, builder.getValuesCount());
assertEquals("/data/zk=a", builder.getValues(0).getColumns(0));
assertEquals("", builder.getValues(0).getColumns(1));
}
@Test
void testCollectBlankCmdKeepsResponseTimeRow() {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
Metrics metrics = telnetMetrics("", List.of("responseTime"));
try (MockedConstruction<TelnetClient> mocked =
Mockito.mockConstruction(TelnetClient.class, (telnetClient, context) ->
Mockito.when(telnetClient.isConnected()).thenReturn(true))) {
telnetCollect.collect(builder, metrics);
}
assertEquals(1, builder.getValuesCount());
}
private static Metrics telnetMetrics(String cmd, List<String> aliasFields) {
Metrics metrics = new Metrics();
metrics.setTelnet(TelnetProtocol.builder().timeout("10").port("2181").cmd(cmd).build());
metrics.setAliasFields(aliasFields);
return metrics;
}
private static MockedConstruction<TelnetClient> mockTelnetReply(String reply) {
InputStream inputStream = new ByteArrayInputStream(reply.getBytes(StandardCharsets.UTF_8));
return Mockito.mockConstruction(TelnetClient.class, (telnetClient, context) -> {
Mockito.when(telnetClient.isConnected()).thenReturn(true);
Mockito.when(telnetClient.getOutputStream()).thenReturn(Mockito.mock(OutputStream.class));
Mockito.when(telnetClient.getInputStream()).thenReturn(inputStream);
});
}
@Test
void preCheck() throws IllegalArgumentException {
// metrics is null
@@ -252,11 +252,12 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
if (metrics.getCalculates() == null) {
metrics.setCalculates(Collections.emptyList());
}
List<String> aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList);
// eg: database_pages=Database pages unconventional mapping
Map<String, String> fieldAliasMap = new HashMap<>(8);
Map<String, JexlExpression> fieldExpressionMap = metrics.getCalculates()
.stream()
.map(cal -> transformCal(cal, fieldAliasMap))
.map(cal -> transformCal(cal, fieldAliasMap, aliasFields))
.filter(Objects::nonNull)
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue));
@@ -270,7 +271,6 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair<String, String>) arr[1], (oldValue, newValue) -> newValue));
List<Metrics.Field> fields = metrics.getFields();
List<String> aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList);
Map<String, String> aliasFieldValueMap = new HashMap<>(8);
Map<String, Object> fieldValueMap = new HashMap<>(8);
Map<String, Object> stringTypefieldValueMap = new HashMap<>(8);
@@ -420,13 +420,19 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
* @param fieldAliasMap field alias map
* @return expr
*/
private Object[] transformCal(String cal, Map<String, String> fieldAliasMap) {
private Object[] transformCal(String cal, Map<String, String> fieldAliasMap, List<String> aliasFields) {
int splitIndex = cal.indexOf("=");
if (splitIndex < 0) {
return null;
}
String field = cal.substring(0, splitIndex).trim();
String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#", "#");
// a direct alias reference (RHS must exactly equal an aliasField, no whitespace/case tolerance) is not a formula,
// JEXL parses "[0]" in such paths as array access and silently returns null
if (aliasFields.contains(expressionStr)) {
fieldAliasMap.put(field, expressionStr);
return null;
}
JexlExpression expression;
try {
expression = JexlExpressionRunner.compile(expressionStr);
@@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.dispatch;
import java.util.List;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Test case for {@link MetricsCollect}
*/
class MetricsCollectTest {
@Test
void calculateFieldsMapsIndexedJsonPathAlias() {
Metrics metrics = Metrics.builder()
.name("pods")
.priority((byte) 0)
.fields(List.of(
Metrics.Field.builder().field("pod").type(CommonConstants.TYPE_STRING).build(),
Metrics.Field.builder().field("rc").type(CommonConstants.TYPE_STRING).build()))
.aliasFields(List.of("$.metadata.name", "$.status.containerStatuses[0].restartCount"))
.calculates(List.of(
"pod=$.metadata.name",
"rc=$.status.containerStatuses[0].restartCount"))
.build();
Timeout timeout = mock(Timeout.class);
WheelTimerTask timerTask = mock(WheelTimerTask.class);
when(timeout.task()).thenReturn(timerTask);
when(timerTask.getJob()).thenReturn(Job.builder().build());
MetricsCollect metricsCollect = new MetricsCollect(metrics, timeout, null, "test", List.of());
CollectRep.MetricsData.Builder collectData = CollectRep.MetricsData.newBuilder();
collectData.addValueRow(CollectRep.ValueRow.newBuilder()
.addColumn("pod-a").addColumn("5").build());
collectData.addValueRow(CollectRep.ValueRow.newBuilder()
.addColumn("pod-b-pending").addColumn(CommonConstants.NULL_VALUE).build());
metricsCollect.calculateFields(metrics, collectData);
List<CollectRep.ValueRow> rows = collectData.getValuesList();
assertEquals(2, rows.size());
assertEquals("pod-a", rows.get(0).getColumns(0));
assertEquals("5", rows.get(0).getColumns(1));
assertEquals("pod-b-pending", rows.get(1).getColumns(0));
assertEquals(CommonConstants.NULL_VALUE, rows.get(1).getColumns(1));
}
}
@@ -36,12 +36,16 @@ public final class JsonPathParser {
private static final ParseContext PARSER;
private static final ParseContext ROW_PARSER;
static {
Configuration conf = Configuration.defaultConfiguration()
.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL)
.addOptions(Option.ALWAYS_RETURN_LIST);
CacheProvider.setCache(new LRUCache(128));
PARSER = JsonPath.using(conf);
// a single row legitimately may not contain the queried path
ROW_PARSER = JsonPath.using(conf.addOptions(Option.SUPPRESS_EXCEPTIONS));
}
private JsonPathParser() {
@@ -73,4 +77,18 @@ public final class JsonPathParser {
return PARSER.parse(content).read(jsonPath, typeRef);
}
/**
* use json path to parse one already-parsed row object, missing paths yield an empty list
* @param document parsed json object of a single row
* @param jsonPath jsonPath relative to the row root
* @return matched values, empty list when the path does not exist in this row
*/
public static List<Object> parseRowWithJsonPath(Object document, String jsonPath) {
if (document == null || StringUtils.isEmpty(jsonPath)) {
return Collections.emptyList();
}
List<Object> values = ROW_PARSER.parse(document).read(jsonPath);
return values == null ? Collections.emptyList() : values;
}
}
@@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.util;
import com.jayway.jsonpath.PathNotFoundException;
import java.util.List;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Test case for {@link JsonPathParser}
*/
class JsonPathParserTest {
private static final String ROW_JSON = "{\"metadata\": {\"name\": \"pod-a\"},"
+ " \"status\": {\"phase\": \"Running\","
+ " \"containerStatuses\": [{\"name\": \"c1\", \"ready\": true, \"restartCount\": 5}]}}";
private Object row() {
return JsonPathParser.parseContentWithJsonPath(ROW_JSON, "$").get(0);
}
@Test
void parseRowWithJsonPathReturnsExistingValue() {
List<Object> values = JsonPathParser.parseRowWithJsonPath(row(), "$.status.containerStatuses[0].restartCount");
assertEquals(1, values.size());
assertEquals(5, values.get(0));
}
@Test
void parseRowWithJsonPathReturnsEmptyListWhenPathMissing() {
Object pendingRow = JsonPathParser
.parseContentWithJsonPath("{\"metadata\": {\"name\": \"pod-b\"}, \"status\": {\"phase\": \"Pending\"}}", "$")
.get(0);
List<Object> values = JsonPathParser.parseRowWithJsonPath(pendingRow, "$.status.containerStatuses[0].restartCount");
assertTrue(values.isEmpty());
}
@Test
void parseRowWithJsonPathReturnsAllValuesForWildcard() {
List<Object> values = JsonPathParser.parseRowWithJsonPath(row(), "$.status.containerStatuses[0].*");
assertEquals(3, values.size());
assertTrue(values.contains("c1"));
assertTrue(values.contains(true));
assertTrue(values.contains(5));
}
@Test
void parseContentWithJsonPathStillThrowsWhenPathMissing() {
assertThrows(PathNotFoundException.class,
() -> JsonPathParser.parseContentWithJsonPath(ROW_JSON, "$.spec.nodeName"));
}
@Test
void parseRowWithJsonPathHandlesNullDocumentAndEmptyPath() {
assertTrue(JsonPathParser.parseRowWithJsonPath(null, "$.status").isEmpty());
assertTrue(JsonPathParser.parseRowWithJsonPath(row(), "").isEmpty());
}
}
@@ -54,7 +54,20 @@
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
</exclusion>
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
</exclusion>
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk18on</artifactId>
<version>${bouncycastle.version}</version>
</dependency>
</dependencies>
</project>
@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.nebulagraph;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import com.vesoft.nebula.client.graph.data.CASignedSSLParam;
import com.vesoft.nebula.util.SslUtil;
import java.io.FileWriter;
import java.math.BigInteger;
import java.nio.file.Path;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.cert.X509Certificate;
import java.util.Date;
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class VesoftSslBouncyCastleSmokeTest {
@Test
void vesoftSslUtilWorksWithBouncyCastleJdk18on(@TempDir Path dir) throws Exception {
KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
generator.initialize(2048);
KeyPair keyPair = generator.generateKeyPair();
X500Name subject = new X500Name("CN=hb-3540-smoke");
JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
subject, BigInteger.ONE,
new Date(System.currentTimeMillis() - 60_000),
new Date(System.currentTimeMillis() + 3_600_000),
subject, keyPair.getPublic());
X509Certificate cert = new JcaX509CertificateConverter()
.getCertificate(certBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate())));
Path crt = dir.resolve("smoke.crt");
Path key = dir.resolve("smoke.key");
try (JcaPEMWriter writer = new JcaPEMWriter(new FileWriter(crt.toFile()))) {
writer.writeObject(cert);
}
try (JcaPEMWriter writer = new JcaPEMWriter(new FileWriter(key.toFile()))) {
writer.writeObject(keyPair.getPrivate());
}
CASignedSSLParam param = new CASignedSSLParam(crt.toString(), crt.toString(), key.toString());
assertNotNull(SslUtil.getSSLSocketFactoryWithCA(param));
}
}
@@ -86,6 +86,11 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
*/
private String parseType;
/**
* Charset of the remote command output, default UTF-8
*/
private String charset;
/**
* IP ADDRESS OR DOMAIN NAME OF THE PEER PROXY HOST
*/
@@ -0,0 +1,158 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.collector.collect.basic.http;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
import org.apache.hertzbeat.collector.collect.http.HttpCollectImpl;
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
import org.apache.hertzbeat.collector.dispatch.unit.impl.DataSizeConvert;
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
import org.apache.hertzbeat.collector.util.CollectUtil;
import org.apache.hertzbeat.common.entity.job.Configmap;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.entity.job.Metrics;
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.Timeout;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.util.ResourceUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* Integration test for etcd monitoring functionality.
* Fixture at src/test/resources/http/etcd/metrics.txt is a real capture from a live
* etcd v3.5.17 /metrics endpoint (see app-etcd.yml for the corresponding template).
*/
@Slf4j
@ExtendWith(MockitoExtension.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class EtcdMonitorE2eTest extends AbstractCollectE2eTest {
private static final int MOCK_SERVER_PORT = 52379;
private static final String LOCALHOST = "127.0.0.1";
private static HttpServer mockServer;
@AfterAll
public static void tearDown() {
if (mockServer != null) {
mockServer.stop(0);
}
}
@BeforeEach
public void setUp() throws Exception {
super.setUp();
collect = new HttpCollectImpl();
// the shared harness wires MetricsCollect with an empty unit-convert list,
// but this template relies on B->MB conversion
Timeout convertTimeout = mock(Timeout.class);
WheelTimerTask convertTimerJob = mock(WheelTimerTask.class);
when(convertTimeout.task()).thenReturn(convertTimerJob);
when(convertTimerJob.getJob()).thenReturn(mock(Job.class));
metricsCollect = new MetricsCollect(mock(Metrics.class), convertTimeout,
mock(CollectDataDispatch.class), null, List.of(new DataSizeConvert()));
String metricsResponse = loadResponseFromFile("classpath:http/etcd/metrics.txt");
mockServer = HttpServer.create(new InetSocketAddress(MOCK_SERVER_PORT), 0);
mockServer.setExecutor(null);
mockServer.start();
mockServer.createContext("/metrics", exchange -> sendTextResponse(exchange, metricsResponse));
}
private String loadResponseFromFile(String resourcePath) throws Exception {
return new String(Files.readAllBytes(ResourceUtils.getFile(resourcePath).toPath()));
}
private void sendTextResponse(HttpExchange exchange, String response) throws IOException {
exchange.getResponseHeaders().set("Content-Type", "text/plain");
final byte[] array = response.getBytes(StandardCharsets.UTF_8);
exchange.sendResponseHeaders(200, array.length);
try (OutputStream os = exchange.getResponseBody()) {
os.write(array);
}
}
private static final Map<String, String> EXPECTED_VALUES = Map.of(
"etcd_server_has_leader", "1",
"etcd_mvcc_db_total_size_in_bytes", "0.0195",
"etcd_server_leader_changes_seen_total", "1",
"process_cpu_seconds_total", "42.41",
"process_resident_memory_bytes", "29.9844");
@Test
public void testEtcdMonitor() {
Job etcdJob = appService.getAppDefine("etcd");
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
for (Metrics metricsDef : etcdJob.getMetrics()) {
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef,
!configmapFromPreCollectData.isEmpty() ? configmapFromPreCollectData.get(0) : new HashMap<>());
CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricsDef.getName());
Assertions.assertEquals(EXPECTED_VALUES.get(metricsDef.getName()),
metricsData.getValues().get(0).getColumns(0),
metricsDef.getName() + " collected value mismatch");
configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData);
}
}
@Override
protected Protocol buildProtocol(Metrics metricsDef) {
HttpProtocol protocol = new HttpProtocol();
protocol.setHost(LOCALHOST);
protocol.setPort(String.valueOf(MOCK_SERVER_PORT));
protocol.setMethod(metricsDef.getHttp().getMethod());
protocol.setParseType(metricsDef.getHttp().getParseType());
protocol.setParseScript(metricsDef.getHttp().getParseScript());
protocol.setUrl(metricsDef.getHttp().getUrl());
return protocol;
}
@Override
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
HttpProtocol protocol = (HttpProtocol) buildProtocol(metricsDef);
metrics.setHttp(protocol);
// prometheus parseType filters by builder.getMetrics(); production sets it in
// MetricsCollect.run() but this test harness does not, so set it here
CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder()
.setMetrics(metricsDef.getName());
return collectMetricsData(metrics, metricsDef, metricsData);
}
}
@@ -0,0 +1,17 @@
# Captured from a live etcd v3.5.17 /metrics endpoint, trimmed to the metric
# families used by app-etcd.yml.
# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes.
# TYPE etcd_mvcc_db_total_size_in_bytes gauge
etcd_mvcc_db_total_size_in_bytes 20480
# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not.
# TYPE etcd_server_has_leader gauge
etcd_server_has_leader 1
# HELP etcd_server_leader_changes_seen_total The number of leader changes seen.
# TYPE etcd_server_leader_changes_seen_total counter
etcd_server_leader_changes_seen_total 1
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 42.41
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 3.1440896e+07
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -86,9 +86,16 @@ resourceRole:
- /api/ingestion/otlp/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -121,10 +128,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -0,0 +1,126 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.manager.config;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import jakarta.servlet.http.HttpServletRequest;
import org.springdoc.core.properties.SwaggerUiConfigProperties;
import org.springdoc.core.properties.SwaggerUiOAuthProperties;
import org.springdoc.core.providers.ObjectMapperProvider;
import org.springdoc.webmvc.ui.SwaggerIndexPageTransformer;
import org.springdoc.webmvc.ui.SwaggerWelcomeCommon;
import org.springframework.core.io.Resource;
import org.springframework.web.servlet.resource.ResourceTransformerChain;
import org.springframework.web.servlet.resource.TransformedResource;
/**
* Adds the HertzBeat access token to same-origin requests made by Swagger UI.
*/
final class AuthorizedSwaggerIndexTransformer extends SwaggerIndexPageTransformer {
private static final String SWAGGER_INITIALIZER = "swagger-initializer.js";
private static final String PRESETS_MARKER = "presets: [";
private static final String INTERCEPTOR_MARKER = "requestInterceptor: (request) => {";
private static final String INTERCEPTOR_RETURN = "return request;";
private static final String AUTHORIZATION_LOGIC = """
const currentUrl = new URL(document.URL);
const requestUrl = new URL(request.url, document.location.origin);
const sameOrigin = currentUrl.protocol === requestUrl.protocol && currentUrl.host === requestUrl.host;
const token = window.localStorage.getItem('Authorization');
if (sameOrigin && token) {
request.headers['Authorization'] = `Bearer ${token}`;
}
""";
private static final String AUTHORIZATION_INTERCEPTOR = """
requestInterceptor: (request) => {
%s
return request;
},
""".formatted(AUTHORIZATION_LOGIC.stripTrailing());
AuthorizedSwaggerIndexTransformer(SwaggerUiConfigProperties swaggerUiConfig,
SwaggerUiOAuthProperties swaggerUiOauthProperties,
SwaggerWelcomeCommon swaggerWelcomeCommon,
ObjectMapperProvider objectMapperProvider) {
super(swaggerUiConfig, swaggerUiOauthProperties, swaggerWelcomeCommon, objectMapperProvider);
}
@Override
public Resource transform(HttpServletRequest request, Resource resource,
ResourceTransformerChain transformerChain) throws IOException {
final Resource transformed = super.transform(request, resource, transformerChain);
if (!SWAGGER_INITIALIZER.equals(resource.getFilename())) {
return transformed;
}
final String initializer;
try (final var input = transformed.getInputStream()) {
initializer = new String(input.readAllBytes(), StandardCharsets.UTF_8);
}
return new TransformedResource(transformed,
addAuthorizationInterceptor(initializer).getBytes(StandardCharsets.UTF_8));
}
/**
* Springdoc writes an interceptor of its own when csrf support is turned on. Two keys
* of the same name would silently drop one of them, so the token is appended to the
* body already there. It goes in at the end because the last write to a header wins,
* and springdoc can be configured to write to {@code Authorization} as well. The
* declarations above are named apart from the ones springdoc emits on purpose: the
* two bodies share a scope, so a collision would be a syntax error.
*
* @param initializer the swagger initializer script
* @return the script with the token attached to its outgoing requests
*/
static String addAuthorizationInterceptor(String initializer) {
final int interceptor = initializer.indexOf(INTERCEPTOR_MARKER);
if (interceptor >= 0) {
final int returnStatement = initializer.indexOf(INTERCEPTOR_RETURN,
interceptor + INTERCEPTOR_MARKER.length());
if (returnStatement < 0) {
throw new IllegalStateException("the swagger initializer interceptor no longer returns the request");
}
return insertLineBefore(initializer, returnStatement, AUTHORIZATION_LOGIC);
}
final int presets = initializer.indexOf(PRESETS_MARKER);
if (presets < 0) {
throw new IllegalStateException("the swagger initializer no longer contains the presets marker");
}
return insertLineBefore(initializer, presets, AUTHORIZATION_INTERCEPTOR);
}
/**
* @param initializer the swagger initializer script
* @param index an index into the line to insert in front of
* @param insertion the lines to insert, newline terminated
* @return the script with the insertion on its own lines, leaving the indentation of
* the line at {@code index} alone
*/
private static String insertLineBefore(String initializer, int index, String insertion) {
final int lineStart = initializer.lastIndexOf('\n', index) + 1;
return initializer.substring(0, lineStart) + insertion + initializer.substring(lineStart);
}
}
@@ -35,7 +35,9 @@ public class SecurityCorsConfiguration {
public FilterRegistrationBean corsFilter() {
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
CorsConfiguration corsConfiguration = new CorsConfiguration();
corsConfiguration.setAllowCredentials(true);
// Requests authenticate with a token in the Authorization header rather than a
// cookie, so no request relies on ambient credentials being sent cross origin.
corsConfiguration.setAllowCredentials(false);
corsConfiguration.setAllowedOriginPatterns(Collections.singletonList(CorsConfiguration.ALL));
corsConfiguration.addAllowedHeader(CorsConfiguration.ALL);
corsConfiguration.addAllowedMethod(CorsConfiguration.ALL);
@@ -25,6 +25,13 @@ import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.info.License;
import io.swagger.v3.oas.models.security.SecurityRequirement;
import io.swagger.v3.oas.models.security.SecurityScheme;
import org.springdoc.core.properties.SwaggerUiConfigProperties;
import org.springdoc.core.properties.SwaggerUiOAuthProperties;
import org.springdoc.core.providers.ObjectMapperProvider;
import org.springdoc.webmvc.ui.SwaggerIndexTransformer;
import org.springdoc.webmvc.ui.SwaggerWelcomeCommon;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -37,6 +44,27 @@ public class SwaggerConfig {
private static final String SECURITY_SCHEME_NAME = "BearerAuth";
/**
* The springdoc beans this one is built from only exist while both switches are on:
* its own ui configuration is conditional on {@code SpringDocConfiguration}, which
* {@code springdoc.api-docs.enabled} gates in turn. Matching both switches keeps a
* deployment that turns the document off from failing to start.
*
* @return the swagger ui index transformer, replacing the springdoc default
*/
@Bean
@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET)
@ConditionalOnProperty(name = {"springdoc.api-docs.enabled", "springdoc.swagger-ui.enabled"},
havingValue = "true", matchIfMissing = true)
public SwaggerIndexTransformer authorizedSwaggerIndexTransformer(
SwaggerUiConfigProperties swaggerUiConfig,
SwaggerUiOAuthProperties swaggerUiOauthProperties,
SwaggerWelcomeCommon swaggerWelcomeCommon,
ObjectMapperProvider objectMapperProvider) {
return new AuthorizedSwaggerIndexTransformer(swaggerUiConfig, swaggerUiOauthProperties,
swaggerWelcomeCommon, objectMapperProvider);
}
@Bean
public OpenAPI springOpenApi() {
return new OpenAPI()
@@ -0,0 +1,328 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The monitoring type categoryservice-application service monitoring db-database monitoring custom-custom monitoring os-operating system monitoring mid-middleware
category: mid
# The monitoring type eg: linux windows tomcat mysql aws...
app: etcd
# The monitoring i18n name
name:
zh-CN: etcd
en-US: etcd
ja-JP: etcd
# The description and help of this monitoring type
help:
zh-CN: HertzBeat 通过调用 <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>etcd Prometheus Metrics 接口</a>(默认在客户端端口 <i>2379</i> 的 <i>/metrics</i> 路径)对 etcd 键值存储(3.4+)的领导者状态、数据库大小、进程资源等指标进行采集监控。<br>您可以点击“<i>新建 etcd</i>”并配置 HOST 端口等相关参数进行添加。<br><span class='help_module_span'>⚠️注意:请确保 HertzBeat 能访问 etcd 的 /metrics 接口。该接口默认由 client listener 提供;若 etcd 仅监听 localhost 或客户端启用了双向 TLS,请通过 --listen-metrics-urls 配置独立的 metrics 地址。</span>
en-US: HertzBeat monitors the etcd key-value store's (3.4+) leader status, database size and process resource usage by calling the <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>etcd Prometheus Metrics endpoint</a> (default at the <i>/metrics</i> path on the client port <i>2379</i>).<br>You can click "<i>New etcd</i>" and configure the host, port and other related params to add it.<br><span class='help_module_span'>Note - make sure HertzBeat can reach etcd's /metrics endpoint. It is served on the client listener by default; if etcd only listens on localhost or client mutual TLS is enabled, configure a dedicated metrics address via --listen-metrics-urls.</span>
zh-TW: HertzBeat 透過調用 <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>etcd Prometheus Metrics 介面</a>(預設在客戶端連接埠 <i>2379</i> 的 <i>/metrics</i> 路徑)對 etcd 鍵值儲存(3.4+)的領導者狀態、資料庫大小、程序資源等指標進行採集監控。<br>您可以點擊“<i>新建 etcd</i>”並配置 HOST 連接埠等相關參數進行添加。<br><span class='help_module_span'>⚠️注意:請確保 HertzBeat 能訪問 etcd 的 /metrics 介面。該介面預設由 client listener 提供;若 etcd 僅監聽 localhost 或客戶端啟用了雙向 TLS,請透過 --listen-metrics-urls 配置獨立的 metrics 地址。</span>
ja-JP: HertzBeat は etcd3.4+)が公開する <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>Prometheus metrics エンドポイント</a>(デフォルトではクライアントポート <i>2379</i> の <i>/metrics</i> パス)から、リーダー状態・データベースサイズ・プロセスリソース等の指標を収集し、etcd キーバリューストアを監視します。<br>「<i>新規 etcd</i>」をクリックしてホストやポートなどのパラメータを設定して追加できます。<br><span class='help_module_span'>⚠️注意:HertzBeat が etcd の /metrics エンドポイントへ到達できることを確認してください。デフォルトでは client listener が提供しますが、etcd が localhost のみを監視している場合やクライアント相互 TLS が有効な場合は、--listen-metrics-urls で専用の metrics アドレスを設定してください。</span>
helpLink:
zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/etcd/
en-US: https://hertzbeat.apache.org/docs/help/etcd/
# Input params define for monitoring(render web ui by the definition)
params:
# field-param field key
- field: host
# name-param field display i18n name
name:
zh-CN: 目标Host
en-US: Target Host
ja-JP: 目標ホスト
# type-param field type(most mapping the html input type)
type: host
# required-true or false
required: true
- field: port
name:
zh-CN: 端口
en-US: Port
ja-JP: ポート
# type-param field type(most mapping the html input type)
type: number
# when type is number, range is required
range: '[0,65535]'
# default value: etcd client port that serves /metrics
defaultValue: 2379
required: true
- field: timeout
name:
zh-CN: 查询超时时间
en-US: Query Timeout
ja-JP: クエリタイムアウト
type: number
required: false
# hide param-true or false
hide: true
defaultValue: 6000
# field-param field key
- field: ssl
# name-param field display i18n name
name:
zh-CN: 启用HTTPS
en-US: HTTPS
ja-JP: HTTPS
# type-param field type(most mapping the html input type)
type: boolean
hide: true
# field-param field key
- field: headers
# name-param field display i18n name
name:
zh-CN: 请求Headers
en-US: Headers
ja-JP: ヘッダ
# type-param field type(most mapping the html input type)
type: key-value
# required-true or false
required: false
hide: true
# when type is key-value, use keyAlias to config key alias name
keyAlias: Header Name
# when type is key-value, use valueAlias to config value alias name
valueAlias: Header Value
# field-param field key
- field: authType
# name-param field display i18n name
name:
zh-CN: 认证方式
en-US: Auth Type
ja-JP: 認証方法
# type-param field type(most mapping the html input type)
type: radio
# required-true or false
required: false
# hide param-true or false
hide: true
# when type is radio checkbox, use option to show optional values {name1:value1,name2:value2}
options:
- label: Basic Auth
value: Basic Auth
- label: Digest Auth
value: Digest Auth
# field-param field key
- field: username
# name-param field display i18n name
name:
zh-CN: 用户名
en-US: Username
ja-JP: ユーザー名
# type-param field type(most mapping the html input type)
type: text
# when type is text, use limit to limit string length
limit: 50
# required-true or false
required: false
# hide param-true or false
hide: true
# field-param field key
- field: password
# name-param field display i18n name
name:
zh-CN: 密码
en-US: Password
ja-JP: パスワード
# type-param field type(most mapping the html input type)
type: password
# required-true or false
required: false
# hide param-true or false
hide: true
# collect metrics config list
# each metrics group name must exactly match a Prometheus metric family name exposed by etcd /metrics
metrics:
# metrics - etcd_server_has_leader (availability: whether this etcd member has a raft leader)
- name: etcd_server_has_leader
i18n:
zh-CN: 领导者状态
en-US: Leader Status
ja-JP: リーダー状態
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 0
fields:
- field: hasLeader
type: 0
i18n:
zh-CN: 是否存在领导者(1有0无)
en-US: Has Leader(1=yes 0=no)
ja-JP: リーダー有無(1=有 0=無)
aliasFields:
- value
calculates:
- hasLeader=value
protocol: http
http:
host: ^_^host^_^
port: ^_^port^_^
url: /metrics
timeout: ^_^timeout^_^
ssl: ^_^ssl^_^
method: GET
headers:
^_^headers^_^: ^_^headers^_^
authorization:
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: prometheus
# metrics - etcd_mvcc_db_total_size_in_bytes
- name: etcd_mvcc_db_total_size_in_bytes
i18n:
zh-CN: 数据库大小
en-US: Database Size
ja-JP: データベースサイズ
priority: 1
fields:
- field: dbSize
type: 0
unit: MB
i18n:
zh-CN: 物理分配的数据库大小
en-US: Physically Allocated DB Size
ja-JP: 物理割当済みDBサイズ
aliasFields:
- value
calculates:
- dbSize=value
units:
- dbSize=B->MB
protocol: http
http:
host: ^_^host^_^
port: ^_^port^_^
url: /metrics
timeout: ^_^timeout^_^
ssl: ^_^ssl^_^
method: GET
headers:
^_^headers^_^: ^_^headers^_^
authorization:
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: prometheus
# metrics - etcd_server_leader_changes_seen_total
- name: etcd_server_leader_changes_seen_total
i18n:
zh-CN: 领导者变更次数
en-US: Leader Changes
ja-JP: リーダー変更回数
priority: 1
fields:
- field: leaderChanges
type: 0
i18n:
zh-CN: 已观测到的领导者变更总次数
en-US: Total Leader Changes Seen
ja-JP: 観測されたリーダー変更総数
aliasFields:
- value
calculates:
- leaderChanges=value
protocol: http
http:
host: ^_^host^_^
port: ^_^port^_^
url: /metrics
timeout: ^_^timeout^_^
ssl: ^_^ssl^_^
method: GET
headers:
^_^headers^_^: ^_^headers^_^
authorization:
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: prometheus
# metrics - process_cpu_seconds_total (standard Go/Prometheus client process collector, always present)
- name: process_cpu_seconds_total
i18n:
zh-CN: 进程CPU时间
en-US: Process CPU Time
ja-JP: プロセスCPU時間
priority: 1
fields:
- field: cpuSeconds
type: 0
unit: s
i18n:
zh-CN: 累计用户与系统CPU时间
en-US: Total User+System CPU Time
ja-JP: 累計ユーザー+システムCPU時間
aliasFields:
- value
calculates:
- cpuSeconds=value
protocol: http
http:
host: ^_^host^_^
port: ^_^port^_^
url: /metrics
timeout: ^_^timeout^_^
ssl: ^_^ssl^_^
method: GET
headers:
^_^headers^_^: ^_^headers^_^
authorization:
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: prometheus
# metrics - process_resident_memory_bytes
- name: process_resident_memory_bytes
i18n:
zh-CN: 进程内存占用
en-US: Process Resident Memory
ja-JP: プロセス常駐メモリ
priority: 1
fields:
- field: memory
type: 0
unit: MB
i18n:
zh-CN: 常驻内存大小
en-US: Resident Memory Size
ja-JP: 常駐メモリサイズ
aliasFields:
- value
calculates:
- memory=value
units:
- memory=B->MB
protocol: http
http:
host: ^_^host^_^
port: ^_^port^_^
url: /metrics
timeout: ^_^timeout^_^
ssl: ^_^ssl^_^
method: GET
headers:
^_^headers^_^: ^_^headers^_^
authorization:
type: ^_^authType^_^
basicAuthUsername: ^_^username^_^
basicAuthPassword: ^_^password^_^
digestAuthUsername: ^_^username^_^
digestAuthPassword: ^_^password^_^
parseType: prometheus
@@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
/**
* Test case for {@link AuthorizedSwaggerIndexTransformer}.
*/
class AuthorizedSwaggerIndexTransformerTest {
@Test
void shouldAttachTheHertzBeatTokenToSameOriginRequests() {
final String initializer = """
window.ui = SwaggerUIBundle({
configUrl: "/v3/api-docs/swagger-config",
presets: [SwaggerUIBundle.presets.apis]
});
""";
final String transformed = AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer);
assertTrue(transformed.contains("window.localStorage.getItem('Authorization')"));
assertTrue(transformed.contains("request.headers['Authorization'] = `Bearer ${token}`"));
assertTrue(transformed.contains("sameOrigin && token"));
assertTrue(transformed.contains("presets: [SwaggerUIBundle.presets.apis]"));
}
@Test
void shouldFailClosedWhenTheSwaggerInitializerShapeChanges() {
assertThrows(IllegalStateException.class,
() -> AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor("window.ui = {};"));
}
/**
* Springdoc writes an interceptor of its own when csrf support is turned on, and it
* can be configured to write to the {@code Authorization} header too. The last write
* wins, so ours has to come after the one already there.
*/
@Test
void shouldComposeWithAnExistingRequestInterceptor() {
final String initializer = """
window.ui = SwaggerUIBundle({
requestInterceptor: (request) => {
request.headers['Authorization'] = 'csrf';
return request;
},
presets: [SwaggerUIBundle.presets.apis]
});
""";
final String transformed = AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer);
assertTrue(transformed.contains("request.headers['Authorization'] = 'csrf'"));
assertTrue(transformed.indexOf("request.headers['Authorization'] = `Bearer ${token}`")
> transformed.indexOf("request.headers['Authorization'] = 'csrf'"),
"the token has to be written after the interceptor that was already there");
assertEquals(1, countOf(transformed, "requestInterceptor:"),
"a second key of the same name would drop one of the two interceptors");
}
@Test
void shouldFailClosedWhenTheExistingInterceptorDoesNotReturnTheRequest() {
final String initializer = """
window.ui = SwaggerUIBundle({
requestInterceptor: (request) => {
presets: [SwaggerUIBundle.presets.apis]
""";
assertThrows(IllegalStateException.class,
() -> AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer));
}
/**
* @param initializer the transformed script
* @param token the substring to count
* @return how many times the substring occurs
*/
private static int countOf(String initializer, String token) {
int count = 0;
for (int index = initializer.indexOf(token); index >= 0; index = initializer.indexOf(token, index + 1)) {
count++;
}
return count;
}
}
@@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import jakarta.servlet.Filter;
import org.junit.jupiter.api.Test;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.mock.web.MockHttpServletResponse;
import org.springframework.mock.web.MockFilterChain;
/**
* Test case for {@link SecurityCorsConfiguration}.
*
* <p>The filter answers every origin, which is intentional, and requests authenticate with
* a token in the Authorization header rather than a cookie, so credentials do not need to
* be allowed. Both halves are asserted: the credentials header is not sent, and a preflight
* still succeeds so the api stays reachable cross origin.
*/
class SecurityCorsConfigurationTest {
private static final String OTHER_ORIGIN = "https://other.example";
@Test
void testCredentialsAreNotAllowedForCrossOriginRequests() throws Exception {
MockHttpServletResponse response = handlePreflight();
assertNotEquals("true", response.getHeader("Access-Control-Allow-Credentials"));
}
@Test
void testCrossOriginRequestsAreStillAnswered() throws Exception {
MockHttpServletResponse response = handlePreflight();
assertNotNull(response.getHeader("Access-Control-Allow-Origin"),
"the api is meant to stay reachable cross origin");
assertEquals(200, response.getStatus());
}
private MockHttpServletResponse handlePreflight() throws Exception {
FilterRegistrationBean<?> registration = new SecurityCorsConfiguration().corsFilter();
Filter filter = (Filter) registration.getFilter();
MockHttpServletRequest request = new MockHttpServletRequest("OPTIONS", "/api/monitors");
request.addHeader("Origin", OTHER_ORIGIN);
request.addHeader("Access-Control-Request-Method", "GET");
MockHttpServletResponse response = new MockHttpServletResponse();
filter.doFilter(request, response, new MockFilterChain());
return response;
}
}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -86,9 +86,16 @@ resourceRole:
- /api/ingestion/otlp/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -121,10 +128,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -19,13 +19,19 @@
package org.apache.hertzbeat.push.service.impl;
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.concurrent.atomic.AtomicInteger;
import jakarta.annotation.Nullable;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
@@ -37,6 +43,7 @@ import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
import org.apache.hertzbeat.push.dao.PushMonitorDao;
import org.apache.hertzbeat.push.service.PushGatewayService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
/**
@@ -46,26 +53,81 @@ import org.springframework.stereotype.Service;
@Slf4j
@Service
public class PushGatewayServiceImpl implements PushGatewayService {
private static final byte PUSH_MONITOR_TYPE = (byte) 1;
private final CommonDataQueue commonDataQueue;
private final PushMonitorDao pushMonitorDao;
private final Map<String, Long> jobInstanceMap;
public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao) {
private final Map<JobInstance, Long> jobInstanceMap;
/**
* Cap on push monitors created automatically from unknown job/instance pairs.
*
* <p>The route is unauthenticated by design, and every new pair used to persist a
* monitor row and add a `jobInstanceMap` entry that is never removed, so a caller
* iterating over made up names could grow the database and the heap without bound.
* Above the cap an unknown pair is refused while the pairs already known keep working,
* which is why eviction is not used here: evicting a live entry would make the next
* push for that pair create a second monitor for the same job and instance.
*/
private final int maxAutoCreatedMonitors;
/**
* Cap on how many bytes a single push body may carry.
*
* <p>The parser materializes its result in memory, and the servlet container does not bound
* a non form request body, so an independent byte limit is still needed alongside the sample
* limit to keep long names and label values from exhausting the heap.
*/
private final long maxBodyBytes;
/**
* Cap on how many samples a single push body may carry. The parser stops before allocating
* a sample beyond this limit, so a compact body cannot create an unbounded object graph.
*/
private final int maxSamples;
/**
* One entry per pair whose monitor is being created, so that concurrent pushes naming the
* same unknown pair wait for one creation instead of each starting their own. Persistence
* stays outside any shared lock: a slow database would otherwise hold every request for an
* unknown pair, and each of those requests is already holding its parsed samples.
*/
private final Map<JobInstance, CompletableFuture<Long>> monitorCreationMap;
/**
* Successful and in flight creations together, so that the cap is claimed before the
* database write rather than counted after it.
*/
private final AtomicInteger trackedMonitorCount;
public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao,
@Value("${hertzbeat.push.max-auto-created-monitors:10000}") int maxAutoCreatedMonitors,
@Value("${hertzbeat.push.max-body-bytes:5242880}") long maxBodyBytes,
@Value("${hertzbeat.push.max-samples:10000}") int maxSamples) {
if (maxAutoCreatedMonitors < 0 || maxBodyBytes < 0 || maxSamples < 0) {
throw new IllegalArgumentException("push gateway limits must not be negative");
}
this.commonDataQueue = commonDataQueue;
this.pushMonitorDao = pushMonitorDao;
this.maxAutoCreatedMonitors = maxAutoCreatedMonitors;
this.maxBodyBytes = maxBodyBytes;
this.maxSamples = maxSamples;
jobInstanceMap = new ConcurrentHashMap<>();
pushMonitorDao.findMonitorsByType((byte) 1).forEach(monitor ->
jobInstanceMap.put(monitor.getApp() + "_" + monitor.getName(), monitor.getId()));
pushMonitorDao.findMonitorsByType(PUSH_MONITOR_TYPE).forEach(monitor ->
jobInstanceMap.put(new JobInstance(monitor.getApp(), monitor.getName()), monitor.getId()));
monitorCreationMap = new ConcurrentHashMap<>();
trackedMonitorCount = new AtomicInteger(jobInstanceMap.size());
}
@Override
public boolean pushPrometheusMetrics(InputStream inputStream, String job, String instance) {
try {
long curTime = Instant.now().toEpochMilli();
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
final long curTime = Instant.now().toEpochMilli();
final Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(
new BoundedInputStream(inputStream, maxBodyBytes), maxSamples);
if (metricFamilyMap == null) {
log.error("parse prometheus metrics is null, job: {}, instance: {}", job, instance);
return false;
@@ -74,20 +136,11 @@ public class PushGatewayServiceImpl implements PushGatewayService {
if (job != null && instance != null) {
// auto create monitor when job and instance not null
// job is app, instance is the name
id = jobInstanceMap.computeIfAbsent(job + "_" + instance, key -> {
log.info("auto create monitor by prometheus push, job: {}, instance: {}", job, instance);
long monitorId = SnowFlakeIdGenerator.generateId();
Monitor monitor = Monitor.builder()
.id(monitorId)
.app(job)
.name(instance)
.instance(instance)
.type((byte) 1)
.status(CommonConstants.MONITOR_UP_CODE)
.build();
this.pushMonitorDao.save(monitor);
return monitorId;
});
final Long monitorId = resolveMonitorId(new JobInstance(job, instance));
if (monitorId == null) {
return false;
}
id = monitorId;
}
for (Map.Entry<String, MetricFamily> entry : metricFamilyMap.entrySet()) {
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
@@ -110,9 +163,18 @@ public class PushGatewayServiceImpl implements PushGatewayService {
builder.addField(CollectRep.Field.newBuilder().setName("value")
.setType(CommonConstants.TYPE_NUMBER).setLabel(false).build());
}
Map<String, String> labelMap = metric.getLabels()
.stream()
.collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
// A repeated label name is refused rather than resolved: the exposition
// format requires the names of a label set to be unique, and keeping one
// of the values would emit a schema carrying that name twice. Built by
// hand so the refusal is a rejection this method can answer with a
// warning, not the error trace a collector's exception would produce.
Map<String, String> labelMap = new HashMap<>(metric.getLabels().size());
for (MetricFamily.Label label : metric.getLabels()) {
if (labelMap.containsKey(label.getName())) {
throw new DuplicateLabelException(label.getName());
}
labelMap.put(label.getName(), label.getValue());
}
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
for (String field : metricsFields) {
String fieldValue = labelMap.get(field);
@@ -125,9 +187,210 @@ public class PushGatewayServiceImpl implements PushGatewayService {
}
}
return true;
} catch (BodyTooLargeException e) {
// A rejection, not a failure: caught apart from the generic handler below so that a
// caller repeating oversized bodies costs one warning line each, not a stack trace
log.warn("reject prometheus push over the {} byte body limit, job: {}, instance: {}",
maxBodyBytes, job, instance);
return false;
} catch (OnlineParser.SampleLimitExceededException e) {
log.warn("reject prometheus push over the {} sample limit, job: {}, instance: {}",
maxSamples, job, instance);
return false;
} catch (DuplicateLabelException e) {
log.warn("reject prometheus push repeating a label name, job: {}, instance: {}: {}",
job, instance, e.getMessage());
return false;
} catch (Exception e) {
log.error("push prometheus metrics error", e);
return false;
}
}
/**
* Returns the monitor id a job/instance pair resolves to, creating the monitor on first
* sight, or null once {@link #maxAutoCreatedMonitors} is reached.
*
* <p>Concurrent pushes naming the same unknown pair share one creation through a future, and
* the cap is claimed by an atomic count before the database write. Nothing here holds a lock
* across that write: unrelated pairs persist in parallel, and a slow database delays only the
* requests naming the pair being created, which matters because every waiting request is
* holding the samples it already parsed.
*
* @param pair Job and instance the push named
* @return The monitor id, or null when the cap leaves no room for a new one
*/
@Nullable
private Long resolveMonitorId(JobInstance pair) {
final Long known = jobInstanceMap.get(pair);
if (known != null) {
return known;
}
final CompletableFuture<Long> proposedCreation = new CompletableFuture<>();
final CompletableFuture<Long> ongoingCreation = monitorCreationMap.putIfAbsent(pair, proposedCreation);
if (ongoingCreation != null) {
try {
return ongoingCreation.join();
} catch (CompletionException e) {
// The request owning the creation reports the failure once; joining its exception
// here would multiply a single database error by every request that waited
return null;
}
}
try {
// Looked up again now that the creation is claimed: another request may have finished
// this pair between the lookup above and this claim, and going on to create it would
// leave two monitors for one pair and spend a second slot of the cap
final Long createdMeanwhile = jobInstanceMap.get(pair);
if (createdMeanwhile != null) {
proposedCreation.complete(createdMeanwhile);
return createdMeanwhile;
}
if (!reserveMonitorSlot()) {
proposedCreation.complete(null);
log.warn("reject prometheus push for unknown job: {}, instance: {}, "
+ "already tracking {} push monitors, limit is {}",
pair.job(), pair.instance(), trackedMonitorCount.get(), maxAutoCreatedMonitors);
return null;
}
boolean created = false;
try {
final long monitorId = createMonitor(pair);
jobInstanceMap.put(pair, monitorId);
created = true;
proposedCreation.complete(monitorId);
return monitorId;
} finally {
if (!created) {
trackedMonitorCount.decrementAndGet();
}
}
} catch (RuntimeException | Error e) {
proposedCreation.completeExceptionally(e);
throw e;
} finally {
monitorCreationMap.remove(pair, proposedCreation);
}
}
/**
* Claims one slot of the cap, or reports that none is left. Claiming before the database
* write is what keeps concurrent creations from exceeding the cap together.
*/
private boolean reserveMonitorSlot() {
int tracked = trackedMonitorCount.get();
while (tracked < maxAutoCreatedMonitors) {
if (trackedMonitorCount.compareAndSet(tracked, tracked + 1)) {
return true;
}
tracked = trackedMonitorCount.get();
}
return false;
}
/**
* Persists a push monitor after its slot of the cap has been claimed.
*/
private long createMonitor(JobInstance pair) {
final String job = pair.job();
final String instance = pair.instance();
log.info("auto create monitor by prometheus push, job: {}, instance: {}", job, instance);
final long monitorId = SnowFlakeIdGenerator.generateId();
final Monitor monitor = Monitor.builder()
.id(monitorId)
.app(job)
.name(instance)
.instance(instance)
.type(PUSH_MONITOR_TYPE)
.status(CommonConstants.MONITOR_UP_CODE)
.build();
this.pushMonitorDao.save(monitor);
return monitorId;
}
/**
* Identifies the monitor a push belongs to.
*
* <p>The two names are kept apart instead of being joined into one string: a separator
* carries no meaning in either name, so `job + "_" + instance` maps ("a", "b_c") and
* ("a_b", "c") onto the same key. Colliding pairs would push their samples into whichever
* monitor was created first, and at startup they would collapse into a single map entry,
* making the cap count fewer monitors than the database actually holds.
*/
private record JobInstance(String job, String instance) {
}
/**
* Raised when a sample repeats a label name, which the exposition format does not allow.
* Kept apart from the generic handler so a malformed body costs one warning line rather
* than an error trace on a route that takes its input from anyone.
*/
static final class DuplicateLabelException extends IOException {
DuplicateLabelException(String name) {
super("sample repeats the label name " + name);
}
}
/**
* Raised when a body goes past {@link #maxBodyBytes}. It is kept apart from the other read
* failures so the caller can answer a body that is merely too large without an error trace.
*/
static final class BodyTooLargeException extends IOException {
BodyTooLargeException(long limit) {
super("push body exceeds the " + limit + " byte limit");
}
}
/**
* Fails the read once the body has delivered more than {@code limit} bytes, instead of
* letting the parser accumulate an unbounded body in memory. Reading stops at the
* failure, so the bytes beyond the limit are never buffered.
*/
static final class BoundedInputStream extends InputStream {
private final InputStream delegate;
private final long limit;
private long bytesRead;
BoundedInputStream(InputStream delegate, long limit) {
this.delegate = delegate;
this.limit = limit;
}
@Override
public int read() throws IOException {
final int value = delegate.read();
if (value != -1) {
recordBytesRead(1);
}
return value;
}
@Override
public int read(byte[] buffer, int offset, int length) throws IOException {
final int bytesReadNow = delegate.read(buffer, offset, length);
if (bytesReadNow > 0) {
recordBytesRead(bytesReadNow);
}
return bytesReadNow;
}
private void recordBytesRead(int increment) throws IOException {
bytesRead += increment;
if (bytesRead > limit) {
throw new BodyTooLargeException(limit);
}
}
@Override
public void close() throws IOException {
delegate.close();
}
}
}
@@ -0,0 +1,367 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.push.service.impl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.push.dao.PushMonitorDao;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Test case for {@link PushGatewayServiceImpl}.
*
* <p>`/api/push/prometheus/**` is unauthenticated by design, so the resource a single
* anonymous request may consume has to be bounded: the body it may carry, the samples it
* may enqueue, and the number of push monitors it may bring into existence.
*/
@ExtendWith(MockitoExtension.class)
class PushGatewayServiceImplTest {
private static final String BODY = "sample_metric{label=\"a\"} 1\n";
@Mock
private CommonDataQueue commonDataQueue;
@Mock
private PushMonitorDao pushMonitorDao;
@BeforeEach
void setUp() {
// The stream test below builds no service, so this default must not be strict
lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of());
}
private PushGatewayServiceImpl createService(int maxMonitors, long maxBodyBytes, int maxSamples) {
return new PushGatewayServiceImpl(commonDataQueue, pushMonitorDao, maxMonitors, maxBodyBytes, maxSamples);
}
private static ByteArrayInputStream createBody(String content) {
return new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
}
/**
* The exposition format requires the names of a label set to be unique, so a sample that
* repeats one is refused rather than resolved. It stays a rejection though: answering it
* with an error trace would let a malformed body fill the log on an anonymous route.
*/
@Test
void testSampleRepeatingTheLabelNameIsRejected() {
final PushGatewayServiceImpl service = createService(10, 1024, 100);
assertFalse(service.pushPrometheusMetrics(
createBody("sample_metric{label=\"a\",label=\"b\"} 1\n"), "job1", "instance1"));
verify(commonDataQueue, never()).sendMetricsData(any());
}
@Test
void testPushIsAcceptedWithinTheLimits() {
final PushGatewayServiceImpl service = createService(10, 1024, 100);
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
verify(pushMonitorDao).save(any(Monitor.class));
}
@Test
void testBodyBeyondTheByteLimitIsRejected() {
final PushGatewayServiceImpl service = createService(10, 16, 100);
assertFalse(service.pushPrometheusMetrics(createBody(BODY.repeat(100)), "job1", "instance1"));
verify(pushMonitorDao, never()).save(any(Monitor.class));
}
@Test
void testBodyBeyondTheSampleLimitIsRejected() {
final PushGatewayServiceImpl service = createService(10, 1024 * 1024, 2);
final StringBuilder many = new StringBuilder();
for (int index = 0; index < 10; index++) {
many.append("sample_metric{label=\"value").append(index).append("\"} 1\n");
}
final ByteArrayInputStream inputStream = createBody(many.toString());
assertFalse(service.pushPrometheusMetrics(inputStream, "job1", "instance1"));
assertTrue(inputStream.available() > 0, "the parser should stop before consuming the remaining samples");
verify(pushMonitorDao, never()).save(any(Monitor.class));
}
@Test
void testBodyAtTheSampleLimitIsAccepted() {
final PushGatewayServiceImpl service = createService(10, 1024, 2);
final String twoSamples = "sample_metric{label=\"a\"} 1\n"
+ "sample_metric{label=\"b\"} 2\n";
assertTrue(service.pushPrometheusMetrics(createBody(twoSamples), "job1", "instance1"));
}
/**
* An unknown job/instance pair persists a monitor row and adds a map entry that is
* never removed, so without a cap an anonymous caller iterating over made up names
* grows the database and the heap without bound.
*/
@Test
void testAutoCreationStopsAtTheMonitorLimit() {
final PushGatewayServiceImpl service = createService(2, 1024, 100);
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job3", "instance3"));
verify(pushMonitorDao, times(2)).save(any(Monitor.class));
}
/**
* The cap must not turn into eviction: a pair already known has to keep resolving to
* the monitor it created, otherwise a later push would create a second monitor for the
* same job and instance.
*/
@Test
void testKnownPairsKeepWorkingAtTheLimit() {
final PushGatewayServiceImpl service = createService(1, 1024, 100);
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "other", "instance"));
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
verify(pushMonitorDao, times(1)).save(any(Monitor.class));
}
/**
* The route is anonymous, so nothing stops a caller from sending its unknown pairs all at
* once. Testing the cap and claiming the entry in two steps lets every request that already
* passed the test create a monitor of its own, which is the cap being exceeded by as many
* requests as the container serves in parallel.
*/
@Test
void testConcurrentPushesForUnknownPairsStopAtTheMonitorLimit() throws Exception {
final int callers = 16;
final PushGatewayServiceImpl service = createService(1, 1024, 100);
final CyclicBarrier startTogether = new CyclicBarrier(callers);
final ExecutorService pool = Executors.newFixedThreadPool(callers);
final AtomicInteger accepted = new AtomicInteger();
try {
final List<Future<?>> pushes = new ArrayList<>();
for (int index = 0; index < callers; index++) {
final String instance = "instance" + index;
pushes.add(pool.submit(() -> {
startTogether.await();
if (service.pushPrometheusMetrics(createBody(BODY), "job", instance)) {
accepted.incrementAndGet();
}
return null;
}));
}
for (final Future<?> push : pushes) {
push.get(30, TimeUnit.SECONDS);
}
} finally {
pool.shutdownNow();
}
assertEquals(1, accepted.get());
verify(pushMonitorDao, times(1)).save(any(Monitor.class));
}
@Test
void testFailedSaveDoesNotConsumeTheMonitorLimit() {
when(pushMonitorDao.save(any(Monitor.class)))
.thenThrow(new IllegalStateException("database unavailable"))
.thenAnswer(invocation -> invocation.getArgument(0));
final PushGatewayServiceImpl service = createService(1, 1024, 100);
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
verify(pushMonitorDao, times(2)).save(any(Monitor.class));
}
@Test
void testNegativeLimitsAreRejectedAtConstruction() {
assertThrows(IllegalArgumentException.class, () -> createService(-1, 1024, 100));
assertThrows(IllegalArgumentException.class, () -> createService(1, -1, 100));
assertThrows(IllegalArgumentException.class, () -> createService(1, 1024, -1));
}
@Test
void testConcurrentPushesForTheSamePairCreateOneMonitor() throws Exception {
final int callers = 8;
final CyclicBarrier startTogether = new CyclicBarrier(callers);
final PushGatewayServiceImpl service = createService(1, 1024, 100);
final ExecutorService pool = Executors.newFixedThreadPool(callers);
try {
final List<Future<Boolean>> pushes = new ArrayList<>();
for (int index = 0; index < callers; index++) {
pushes.add(pool.submit(() -> {
startTogether.await();
return service.pushPrometheusMetrics(createBody(BODY), "job", "instance");
}));
}
for (final Future<Boolean> push : pushes) {
assertTrue(push.get(30, TimeUnit.SECONDS));
}
} finally {
pool.shutdownNow();
}
verify(pushMonitorDao).save(any(Monitor.class));
}
/**
* A request may read no entry for a pair and only then claim the creation, by which time
* another request may have created that pair and cleared its claim. Without a second lookup
* once the claim is won, this request goes on to create the same pair again, leaving two
* monitors for one pair and a slot of the cap spent for good.
*
* <p>The interleaving is forced rather than raced: the map hands back a miss, and while it
* does, a competing push runs its whole creation.
*/
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
void testStaleMissDoesNotCreateTwoMonitorsForOnePair() throws Exception {
final PushGatewayServiceImpl service = createService(2, 1024, 100);
final Field trackedPairs = PushGatewayServiceImpl.class.getDeclaredField("jobInstanceMap");
trackedPairs.setAccessible(true);
final AtomicBoolean competed = new AtomicBoolean();
final Map probing = new ConcurrentHashMap() {
@Override
public Object get(Object key) {
final Object value = super.get(key);
if (value == null && competed.compareAndSet(false, true)) {
service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1");
}
return value;
}
};
trackedPairs.set(service, probing);
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
verify(pushMonitorDao, times(1)).save(any(Monitor.class));
assertEquals(1, probing.size());
// The slot the duplicate would have taken is still there for another pair
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
}
@Test
void testMonitorsLoadedAtStartupCountTowardsTheLimit() {
lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of(
Monitor.builder().id(1L).app("job1").name("instance1").build()));
final PushGatewayServiceImpl service = createService(1, 1024, 100);
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
verify(pushMonitorDao, never()).save(any(Monitor.class));
}
/**
* A separator carries no meaning inside a job or an instance name, so the two names must not
* be joined into a single key: ("job", "a_b") and ("job_a", "b") are different monitors, and
* the second pair must not push its samples into the monitor the first one created.
*/
@Test
void testPairsSharingTheSeparatorAreDistinctMonitors() {
final PushGatewayServiceImpl service = createService(10, 1024, 100);
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job", "a_b"));
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job_a", "b"));
verify(pushMonitorDao, times(2)).save(any(Monitor.class));
final ArgumentCaptor<CollectRep.MetricsData> pushed =
ArgumentCaptor.forClass(CollectRep.MetricsData.class);
verify(commonDataQueue, times(2)).sendMetricsData(pushed.capture());
assertNotEquals(pushed.getAllValues().get(0).getId(), pushed.getAllValues().get(1).getId());
}
/**
* Colliding pairs must not collapse into one entry while the monitors are loaded either,
* which would let the cap count fewer monitors than the database actually holds.
*/
@Test
void testPairsSharingTheSeparatorCountSeparatelyAtStartup() {
lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of(
Monitor.builder().id(1L).app("job").name("a_b").build(),
Monitor.builder().id(2L).app("job_a").name("b").build()));
final PushGatewayServiceImpl service = createService(2, 1024, 100);
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job3", "instance3"));
verify(pushMonitorDao, never()).save(any(Monitor.class));
}
@Test
void testBoundedStreamStopsAtTheLimit() throws Exception {
final PushGatewayServiceImpl.BoundedInputStream stream =
new PushGatewayServiceImpl.BoundedInputStream(createBody("abcdef"), 3);
assertEquals('a', stream.read());
assertEquals('b', stream.read());
assertEquals('c', stream.read());
assertThrows(PushGatewayServiceImpl.BodyTooLargeException.class, stream::read);
}
/**
* A body over the limit is a rejection rather than a failure, so it must be distinguishable
* from a read that genuinely broke: the route is anonymous, and answering every oversized
* body with an error trace lets a caller fill the log at will.
*/
@Test
void testBodyOverTheByteLimitIsRejectedNotFailed() {
final PushGatewayServiceImpl.BoundedInputStream stream =
new PushGatewayServiceImpl.BoundedInputStream(createBody(BODY.repeat(100)), 16);
final IOException raised = assertThrows(IOException.class, stream::readAllBytes);
assertInstanceOf(PushGatewayServiceImpl.BodyTooLargeException.class, raised);
}
}
@@ -71,6 +71,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -373,3 +383,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume.
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -60,6 +60,10 @@ resourceRole:
- /api/collector/**===post===[admin,user]
- /api/collector/**===put===[admin,user]
- /api/collector/**===delete===[admin]
- /api/plugin/**===get===[admin]
- /api/plugin/**===post===[admin]
- /api/plugin/**===put===[admin]
- /api/plugin/**===delete===[admin]
# the secret config holds the jwt signing key and the aes key protecting stored
# credentials, so it stays admin only and is additionally refused by the controller
- /api/config/secret===get===[admin]
@@ -70,6 +74,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -99,11 +120,18 @@ resourceRole:
- /api/account/token===get===[admin]
- /api/account/token/**===post===[admin]
- /api/account/token/**===delete===[admin]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -140,10 +168,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -0,0 +1,142 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.startup.security;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import jakarta.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.yaml.snakeyaml.Yaml;
/**
* Guards the springdoc switches that keep the openapi document off by default.
*
* <p>Scoping the document to the admin role is only half the story. The document is a map
* of every route, http method, parameter and model, so a deployment that has no use for it
* should not serve it at all: both switches are off unless a deployment opts in, and the
* rbac rules stay as the second line of defence for deployments that do. The swagger ui
* page itself is reachable anonymously through the {@code /**}{@code /*.html===get}
* exclusion, but it renders nothing until the caller proves it holds the admin role.
*/
class OpenApiDocumentDisabledByDefaultTest {
private static final Path SCRIPT_DIR = Path.of("..", "script");
private static final List<List<String>> SWITCHES = List.of(
List.of("springdoc", "api-docs", "enabled"),
List.of("springdoc", "swagger-ui", "enabled"));
@Test
void shouldDisableTheOpenApiDocumentInThePackagedConfig() throws IOException {
try (InputStream in = OpenApiDocumentDisabledByDefaultTest.class.getResourceAsStream("/application.yml")) {
assertNotNull(in, "application.yml must be on the classpath");
assertDisabled(documentsOf(in), "application.yml");
}
}
/**
* The deployment scripts ship their own copies of {@code application.yml} and mount
* them over the packaged one, so a switch flipped only in the packaged file would
* still leave every container deployment serving the document.
*/
@Test
void shouldDisableTheOpenApiDocumentInDeploymentCopies() throws IOException {
for (Path copy : deploymentCopies()) {
try (InputStream in = Files.newInputStream(copy)) {
assertDisabled(documentsOf(in), copy.toString());
}
}
}
/**
* Asserts that every declaration of the springdoc switches across a multi document
* yaml turns the endpoint off, and that at least one declaration exists - a file that
* simply omits them falls back to the springdoc default, which is enabled.
*
* @param documents the yaml documents, in the order spring applies them
* @param source the file the documents came from, for the failure message
*/
private static void assertDisabled(List<Map<String, Object>> documents, String source) {
for (List<String> path : SWITCHES) {
List<Object> declarations = documents.stream()
.map(document -> valueAt(document, path))
.filter(Objects::nonNull)
.toList();
assertFalse(declarations.isEmpty(),
String.join(".", path) + " is unset in " + source + ", so it falls back to enabled");
declarations.forEach(declared -> assertEquals(Boolean.FALSE, declared,
String.join(".", path) + " is enabled in " + source));
}
}
/**
* @param document the parsed yaml document
* @param path the key path to walk, outermost first
* @return the value at that path, or null when any segment is missing
*/
@Nullable
private static Object valueAt(Map<String, Object> document, List<String> path) {
Object current = document;
for (String key : path) {
if (!(current instanceof Map<?, ?> map)) {
return null;
}
current = map.get(key);
}
return current;
}
@SuppressWarnings("unchecked")
private static List<Map<String, Object>> documentsOf(InputStream in) {
List<Map<String, Object>> documents = new ArrayList<>();
for (Object document : new Yaml().loadAll(in)) {
if (document instanceof Map<?, ?> map) {
documents.add((Map<String, Object>) map);
}
}
return documents;
}
/**
* @return the {@code application.yml} copies shipped by the deployment scripts
*/
private static Set<Path> deploymentCopies() throws IOException {
assumeTrue(Files.isDirectory(SCRIPT_DIR),
"running outside the source tree, the packaged file asserted above is all we can see");
try (Stream<Path> paths = Files.walk(SCRIPT_DIR)) {
Set<Path> copies = paths.filter(path -> path.getFileName().toString().equals("application.yml"))
.collect(Collectors.toCollection(LinkedHashSet::new));
assertFalse(copies.isEmpty(), "expected the deployment scripts to ship application.yml copies");
return copies;
}
}
}
@@ -0,0 +1,207 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.startup.security;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import com.usthe.sureness.matcher.util.TirePathTree;
import jakarta.annotation.Nullable;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.yaml.snakeyaml.Yaml;
/**
* Guards the rbac rules covering the generated openapi document.
*
* <p>These paths used to sit in {@code excludedResource}. Sureness checks the exclusion
* tree before any credential check, so an anonymous request returned the full document:
* every route, http method, parameter name and type, and every request and response
* model. That is a ready made map of the attack surface, so it is scoped like any other
* administrative resource.
*/
class SurenessOpenApiDocRuleTest {
private static final String SEPARATOR = "===";
/**
* {@code excludedResource} entries are written as {@code api===method}, but
* {@code TirePathTree} only accepts the three segment {@code api===method===roles}
* shape and silently drops anything else. {@code DefaultPathRoleMatcher} therefore
* appends this marker to every excluded rule before it builds the exclusion tree, and
* a test that skips the same step builds an empty tree that matches nothing - every
* {@code assertNull} against it then passes no matter what the yaml says.
*/
private static final String EXCLUDE_ROLE = SEPARATOR + "[exclude]";
private static final Path SCRIPT_DIR = Path.of("..", "script");
private static TirePathTree roleTree;
private static TirePathTree excludeTree;
@BeforeAll
@SuppressWarnings("unchecked")
static void loadSurenessConfig() throws IOException {
List<String> resourceRole;
List<String> excludedResource;
try (InputStream in = SurenessOpenApiDocRuleTest.class.getResourceAsStream("/sureness.yml")) {
assertNotNull(in, "sureness.yml must be on the classpath");
Map<String, Object> document = new Yaml().load(in);
resourceRole = (List<String>) document.get("resourceRole");
excludedResource = (List<String>) document.get("excludedResource");
}
assertNotNull(resourceRole, "resourceRole must be present");
assertNotNull(excludedResource, "excludedResource must be present");
roleTree = new TirePathTree();
roleTree.buildTree(new LinkedHashSet<>(resourceRole));
excludeTree = excludeTreeOf(excludedResource);
}
@Test
void shouldRestrictTheOpenApiDocumentToAdmin() {
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs"));
assertEquals("[admin]", rolesFor(roleTree, "/v2/api-docs"));
assertEquals("[admin]", rolesFor(roleTree, "/swagger-resources/configuration/ui"));
}
/**
* Springdoc also serves the grouped documents and its own config under the same
* prefix; a rule bound to the bare path would leave those anonymous.
*/
@Test
void shouldCoverTheGroupedDocumentsToo() {
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs/swagger-config"));
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs/default"));
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs.yaml/default"));
}
/**
* {@code OpenApiWebMvcResource} maps the document twice, at the configured path and at
* that path with a {@code .yaml} suffix. The suffixed form is a sibling path segment
* rather than a child, so {@code /v3/api-docs/**} does not reach it and it needs its
* own rule. Without one the yaml document carries no role requirement at all, which
* sureness treats as no restriction for any authenticated caller including guest.
*/
@Test
void shouldRestrictTheYamlDocumentToAdmin() {
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs.yaml"));
}
@Test
void shouldStopTreatingTheOpenApiDocumentAsAnonymous() {
assertNull(rolesFor(excludeTree, "/v3/api-docs"));
assertNull(rolesFor(excludeTree, "/v3/api-docs.yaml"));
assertNull(rolesFor(excludeTree, "/v3/api-docs.yaml/default"));
assertNull(rolesFor(excludeTree, "/v3/api-docs/swagger-config"));
assertNull(rolesFor(excludeTree, "/v2/api-docs"));
assertNull(rolesFor(excludeTree, "/swagger-resources/configuration/ui"));
}
/**
* The assertions above are all {@code assertNull}, so they only mean something while
* the exclusion tree is capable of matching at all. This pins a path that is meant to
* stay anonymous and fails if the tree was built in a shape sureness would have
* rejected.
*/
@Test
void shouldKeepMatchingIntentionallyExcludedResources() {
assertEquals("[exclude]", rolesFor(excludeTree, "/api/i18n/lang"));
}
/**
* The deployment scripts ship their own copies of {@code sureness.yml}; a rule fixed
* only in the packaged file would still leave every container deployment serving the
* document to anonymous callers.
*/
@Test
void deploymentCopiesRestrictTheOpenApiDocument() throws IOException {
for (Path copy : deploymentCopies()) {
TirePathTree copyRoleTree = new TirePathTree();
copyRoleTree.buildTree(new LinkedHashSet<>(sectionOf(copy, "resourceRole")));
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs"),
"the openapi document is unruled or over-granted in " + copy);
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs.yaml"),
"the yaml openapi document is unruled or over-granted in " + copy);
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs.yaml/default"),
"the grouped yaml openapi document is unruled or over-granted in " + copy);
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs/swagger-config"),
"the springdoc ui config is unruled or over-granted in " + copy);
TirePathTree copyExcludeTree = excludeTreeOf(sectionOf(copy, "excludedResource"));
assertNull(rolesFor(copyExcludeTree, "/v3/api-docs"),
"the openapi document is still anonymous in " + copy);
assertEquals("[exclude]", rolesFor(copyExcludeTree, "/api/i18n/lang"),
"the exclusion tree matches nothing in " + copy + ", the assertion above proves nothing");
}
}
@Nullable
private static String rolesFor(TirePathTree tree, String path) {
return tree.searchPathFilterRoles(path + SEPARATOR + "get");
}
/**
* @param excludedResource the {@code excludedResource} rules, as written in the yaml
* @return the exclusion tree, built the way {@code DefaultPathRoleMatcher} builds it
*/
private static TirePathTree excludeTreeOf(List<String> excludedResource) {
TirePathTree tree = new TirePathTree();
tree.buildTree(excludedResource.stream()
.map(rule -> rule + EXCLUDE_ROLE)
.collect(Collectors.toCollection(LinkedHashSet::new)));
return tree;
}
/**
* @return the {@code sureness.yml} copies shipped by the deployment scripts
*/
private static Set<Path> deploymentCopies() throws IOException {
assumeTrue(Files.isDirectory(SCRIPT_DIR),
"running outside the source tree, the packaged file asserted above is all we can see");
try (Stream<Path> paths = Files.walk(SCRIPT_DIR)) {
Set<Path> copies = paths.filter(path -> path.getFileName().toString().equals("sureness.yml"))
.collect(Collectors.toCollection(LinkedHashSet::new));
assertFalse(copies.isEmpty(), "expected the deployment scripts to ship sureness.yml copies");
return copies;
}
}
@SuppressWarnings("unchecked")
private static List<String> sectionOf(Path copy, String section) throws IOException {
Map<String, Object> document;
try (InputStream in = Files.newInputStream(copy)) {
document = new Yaml().load(in);
}
List<String> rules = (List<String>) document.get(section);
assertNotNull(rules, section + " must be present in " + copy);
return rules;
}
}
@@ -0,0 +1,130 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.startup.security;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import com.usthe.sureness.matcher.util.TirePathTree;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.yaml.snakeyaml.Yaml;
/**
* Guards the routes that carried no rbac rule at all.
*
* <p>A route absent from {@code sureness.yml} leaves `supportRoles` null, and
* `BaseProcessor.authorized` returns early when no role is required, so every one of
* these was reachable by any authenticated account including {@code guest}: a raw promql
* passthrough to the time series database, log deletion, log and alert injection, label
* management and the internal queue metrics of the hertzbeat process.
*/
class SurenessUnruledEndpointTest {
private static final String SEPARATOR = "===";
private static TirePathTree roleTree;
@BeforeAll
@SuppressWarnings("unchecked")
static void loadSurenessConfig() throws IOException {
List<String> resourceRole;
try (InputStream in = SurenessUnruledEndpointTest.class.getResourceAsStream("/sureness.yml")) {
assertNotNull(in, "sureness.yml must be on the classpath");
Map<String, Object> document = new Yaml().load(in);
resourceRole = (List<String>) document.get("resourceRole");
}
assertNotNull(resourceRole, "resourceRole must be present");
roleTree = new TirePathTree();
roleTree.buildTree(new LinkedHashSet<>(resourceRole));
}
private static String rolesFor(String path, String method) {
return roleTree.searchPathFilterRoles(path + SEPARATOR + method);
}
/**
* `PromqlQueryExecutor` forwards the submitted expression verbatim, so this route reads
* the whole metric store regardless of which monitors the caller may see.
*/
@Test
void queryingTheWarehouseDirectlyIsRestrictedToAdmin() {
assertEquals("[admin]", rolesFor("/api/warehouse/query", "post"));
}
@Test
void probingStorageAvailabilityStaysOpenToEveryRole() {
assertEquals("[admin,user,guest]", rolesFor("/api/warehouse/storage/status", "get"));
}
@Test
void deletingLogsIsRestrictedToAdmin() {
assertEquals("[admin]", rolesFor("/api/logs", "delete"));
}
@Test
void readingLogsStaysOpenToEveryRole() {
assertEquals("[admin,user,guest]", rolesFor("/api/logs/list", "get"));
}
/**
* Matches how the sibling ingestion routes `/api/otlp/**` and `/api/logs/ingest/**`
* are already scoped, so a low privileged account can no longer forge log records.
*/
@Test
void ingestingOtlpLogsRequiresAtLeastUser() {
assertEquals("[admin,user]", rolesFor("/api/logs/otlp/v1/logs", "post"));
}
/**
* The prometheus alertmanager webhook injects alerts, which drive notifications.
* Scoped like the sibling `/api/alerts/report` route.
*/
@Test
void injectingPrometheusAlertsRequiresAtLeastUser() {
assertEquals("[admin,user]", rolesFor("/api/v2/alerts", "post"));
}
@Test
void labelWritesFollowTheUsualScoping() {
assertEquals("[admin,user,guest]", rolesFor("/api/label", "get"));
assertEquals("[admin,user]", rolesFor("/api/label", "post"));
assertEquals("[admin,user]", rolesFor("/api/label", "put"));
assertEquals("[admin]", rolesFor("/api/label", "delete"));
}
@Test
void processQueueMetricsAreRestrictedToAdmin() {
assertEquals("[admin]", rolesFor("/api/metrics", "get"));
}
/**
* Favourites are stored per account and rendered on the monitor pages, so they stay
* reachable by every role even though they sit under the same path prefix.
*/
@Test
void metricFavouritesStayOpenToEveryRole() {
assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1", "get"));
assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1/cpu", "post"));
assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1/cpu", "delete"));
}
}
@@ -66,6 +66,7 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage {
private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://";
private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')");
private static final Pattern NUL_CHAR_PATTERN = Pattern.compile("\\u0000");
private static final String INSTANCE_NULL = "''";
private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s";
private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s";
@@ -346,12 +347,13 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage {
}
private String formatStringValue(String value) {
String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
// bugfix Argument list too long
if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) {
// snmp octet strings may carry NUL padding that breaks the insert sql
String formatValue = NUL_CHAR_PATTERN.matcher(value).replaceAll("");
// truncate the logical value before escaping so the cut cannot split an escape sequence
if (formatValue.length() > tableStrColumnDefineMaxLength) {
formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength);
}
return formatValue;
return SQL_SPECIAL_STRING_PATTERN.matcher(formatValue).replaceAll("\\\\$0");
}
@Override
@@ -115,6 +115,25 @@ class TdEngineDataStorageTest {
assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"), "Should contain timestamp and value 68.7");
}
@Test
void testSaveDataStripsControlCharacters() throws Exception {
tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties);
setPrivateField(tdEngineDataStorage, "hikariDataSource", mockHikariDataSource);
setParentPrivateField(tdEngineDataStorage, "serverAvailable", true);
// snmp octet strings can carry NUL bytes (issue #1481); tabs are legitimate data and stay
CollectRep.MetricsData metricsData = generateMockedMetricsData("Loopback\tInterface 1\u0000");
tdEngineDataStorage.saveData(metricsData);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture());
String executedSql = sqlCaptor.getValue();
assertTrue(executedSql.contains("Loopback\tInterface 1'"), "NUL stripped, tab preserved");
assertTrue(executedSql.indexOf('\u0000') < 0, "no raw NUL byte may reach the sql text");
assertTrue(!executedSql.contains("\\u0000"), "no textual NUL escape may survive in the labels json");
}
@Test
void destroy() {
}
@@ -146,6 +165,10 @@ class TdEngineDataStorageTest {
}
public static CollectRep.MetricsData generateMockedMetricsData() {
return generateMockedMetricsData("test-%server-01");
}
public static CollectRep.MetricsData generateMockedMetricsData(String instanceValue) {
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
when(mockMetricsData.getId()).thenReturn(0L);
@@ -156,9 +179,9 @@ class TdEngineDataStorageTest {
when(mockMetricsData.getInstance()).thenReturn("test-%server-01");
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
List<String> columnValues = List.of("test-%server-01", "68.7");
List<String> columnValues = List.of(instanceValue, "68.7");
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
when(mockValueRow.getColumns(0)).thenReturn("test-%server-01");
when(mockValueRow.getColumns(0)).thenReturn(instanceValue);
when(mockValueRow.getColumns(1)).thenReturn("68.7");
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
@@ -177,7 +200,7 @@ class TdEngineDataStorageTest {
Field instanceArrowField = new Field("instance", instanceFieldType, null);
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
when(instanceCell.getField()).thenReturn(instanceArrowField);
when(instanceCell.getValue()).thenReturn("test-%server-01");
when(instanceCell.getValue()).thenReturn(instanceValue);
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
+65
View File
@@ -0,0 +1,65 @@
---
id: etcd
title: Monitoringetcd monitoring
sidebar_label: etcd
keywords: [open source monitoring tool, open source middleware monitoring tool, monitoring etcd metrics]
---
> HertzBeat monitors the etcd key-value store by collecting metrics from the Prometheus metrics endpoint that etcd exposes.
>
> etcd 3.4+ is supported (the database size metric `etcd_mvcc_db_total_size_in_bytes` replaced the old `etcd_debugging_*` name in 3.4).
## PreRequisites
### Make sure HertzBeat can reach etcd's metrics endpoint
etcd exposes Prometheus-format metrics on its client port (default `2379`) at the `/metrics` path. Make sure this address is reachable from HertzBeat:
1. If etcd only listens on localhost, or client mutual TLS is enabled on the client port, configure a dedicated metrics listener via [`--listen-metrics-urls`](https://etcd.io/docs/latest/op-guide/configuration/). It serves the metrics and health-check endpoints; if exposed without TLS, restrict it to a trusted network.
2. Access `{metrics-host}:{metrics-port}/metrics` (the client port `2379` by default) from the HertzBeat host to confirm metrics data can be fetched.
More information see [etcd monitoring documentation](https://etcd.io/docs/latest/op-guide/monitoring/).
### Configuration parameter
| Parameter name | Parameter help description |
|----------------------|---------------------------------------------------------------------------------------------|
| Target Host | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://) |
| Port | Port of the etcd metrics endpoint, default 2379 when using the client listener |
| Query Timeout | HTTP request timeout in milliseconds, default 6000 |
| HTTPS | Whether to use HTTPS to request the metrics endpoint |
| Headers | Optional extra HTTP request headers |
| Auth Type | Optional Basic/Digest auth if the metrics endpoint sits behind an auth proxy |
| Username / Password | Credentials used when Auth Type is set |
### Collection Metric
#### Metric setetcd_server_has_leader
| Metric name | Metric unit | Metric help description |
|-------------|-------------|---------------------------------------------------------------|
| hasLeader | none | Whether this etcd member has a raft leader (1=yes, 0=no) |
#### Metric setetcd_mvcc_db_total_size_in_bytes
| Metric name | Metric unit | Metric help description |
|-------------|-------------|--------------------------------------------------------|
| dbSize | MB | Total size of the underlying database physically allocated |
#### Metric setetcd_server_leader_changes_seen_total
| Metric name | Metric unit | Metric help description |
|-----------------|-------------|-------------------------------------------|
| leaderChanges | none | Total number of leader changes observed |
#### Metric setprocess_cpu_seconds_total
| Metric name | Metric unit | Metric help description |
|-------------|-------------|---------------------------------------------------|
| cpuSeconds | second | Cumulative user and system CPU time consumed |
#### Metric setprocess_resident_memory_bytes
| Metric name | Metric unit | Metric help description |
|-------------|-------------|-----------------------------------|
| memory | MB | Resident memory size of the process |
+33 -4
View File
@@ -51,6 +51,13 @@ resourceRole:
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
- /api/status/page/**===delete===[admin]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
@@ -82,10 +89,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -140,6 +143,32 @@ account:
role: [user]
```
## OpenAPI Document And Swagger UI
The generated OpenAPI document lists every route, http method, parameter name and type, and every request and response model. It is a ready made map of the attack surface, so HertzBeat does not serve it by default: `springdoc.api-docs.enabled` and `springdoc.swagger-ui.enabled` are both `false` in the shipped `application.yml`, which makes `/v3/api-docs` and `/swagger-ui/index.html` return 404.
If you need the document, opt in by updating the `application.yml` file in the `config` directory:
```yaml
springdoc:
api-docs:
enabled: true
swagger-ui:
enabled: true
```
Once enabled, the document endpoints are still scoped to the `admin` role by the `resourceRole` rules above. Sign in to the HertzBeat web application as an administrator before opening `/swagger-ui/index.html`; the Swagger UI attaches the stored HertzBeat token to its same-origin document and try-it-out requests, and the page loads without asking for anything.
Without that session the document is not exposed, but the page does not fail silently either. `/swagger-ui/index.html` is a static file and still loads; its request for `/v3/api-docs/swagger-config` is answered with `401` and a `WWW-Authenticate: Digest` challenge, so the browser asks for a username and password. Administrator credentials entered there let the document through, and an account without the `admin` role is answered with `403`.
The document can also be fetched directly with an administrator token:
```shell
curl -H "Authorization: Bearer $YOUR_ADMIN_TOKEN" http://localhost:1157/v3/api-docs
```
> ⚠️ Do not move the OpenAPI paths into `excludedResource`; doing so makes the complete document anonymous again.
## Update Security Secret
> This secret is the key for account security encryption management and needs to be updated to your custom key string of the same length.
@@ -0,0 +1,65 @@
---
id: etcd
title: 监控:etcd 监控
sidebar_label: etcd
keywords: [开源监控系统, 中间件监控, etcd监控]
---
> HertzBeat 通过采集 etcd 暴露的 Prometheus metrics 接口数据,对 etcd 键值存储进行监控。
>
> 支持 etcd 3.4 及以上版本(数据库大小指标 `etcd_mvcc_db_total_size_in_bytes` 自 3.4 起替代旧的 `etcd_debugging_*` 命名)。
## 监控前操作
### 确认 HertzBeat 能访问 etcd 的 metrics 接口
etcd 会在客户端端口(默认 `2379`)的 `/metrics` 路径暴露 Prometheus 格式的指标。请确保 HertzBeat 能访问该地址:
1. 若 etcd 仅监听 localhost,或客户端端口启用了双向 TLS,请通过 [`--listen-metrics-urls`](https://etcd.io/docs/latest/op-guide/configuration/) 配置独立的 metrics 监听地址。该地址提供 metrics 与健康检查端点;若不加 TLS 暴露,请仅限受信任网络访问。
2. 从 HertzBeat 所在机器访问 `{metrics-host}:{metrics-port}/metrics`(默认为客户端端口 `2379`),确认能获取到 metrics 数据。
更多信息请参考 [etcd 监控文档](https://etcd.io/docs/latest/op-guide/monitoring/)。
### 配置参数
| 参数名称 | 参数帮助描述 |
|--------|-----------------------------------------------|
| 目标Host | 被监控的对端IPV4,IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 |
| 端口 | metrics 接口端口,使用客户端 listener 时默认为 2379 |
| 查询超时时间 | HTTP请求超时时间,单位毫秒,默认6000 |
| 启用HTTPS | 是否使用 HTTPS 请求 metrics 接口 |
| 请求Headers | 可选的额外 HTTP 请求头 |
| 认证方式 | 若 metrics 接口在认证代理后面,可选 Basic/Digest 认证 |
| 用户名/密码 | 配置认证方式后使用的凭据 |
### 采集指标
#### 指标集合:etcd_server_has_leader
| 指标名称 | 指标单位 | 指标帮助描述 |
|-----------|------|------------------------------|
| hasLeader | 无 | 该 etcd 成员是否存在 raft 领导者(1有0无) |
#### 指标集合:etcd_mvcc_db_total_size_in_bytes
| 指标名称 | 指标单位 | 指标帮助描述 |
|--------|------|-----------------|
| dbSize | MB | 物理分配的数据库总大小 |
#### 指标集合:etcd_server_leader_changes_seen_total
| 指标名称 | 指标单位 | 指标帮助描述 |
|----------------|------|--------------|
| leaderChanges | 无 | 已观测到的领导者变更总次数 |
#### 指标集合:process_cpu_seconds_total
| 指标名称 | 指标单位 | 指标帮助描述 |
|------------|------|------------------|
| cpuSeconds | 秒 | 累计用户与系统CPU使用时间 |
#### 指标集合:process_resident_memory_bytes
| 指标名称 | 指标单位 | 指标帮助描述 |
|--------|------|---------|
| memory | MB | 进程常驻内存大小 |
@@ -52,6 +52,13 @@ resourceRole:
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
- /api/status/page/**===delete===[admin]
# OpenAPI 文档包含全部路由、参数与数据模型,等同于一份接口地图,
# 因此按普通管理类资源收敛到 admin,不再匿名开放
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# 需要被过滤保护的资源,不认证鉴权直接访问
# /api/v1/source3===get 表示 /api/v1/source3===get 可以被任何人访问 无需登录认证鉴权
@@ -82,10 +89,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -141,6 +144,32 @@ account:
role: [user]
```
## OpenAPI 文档与 Swagger UI
生成的 OpenAPI 文档会列出全部路由、HTTP 方法、参数名与类型,以及所有请求和响应模型,等同于一份现成的攻击面地图,因此 HertzBeat 默认不对外提供:随包发布的 `application.yml``springdoc.api-docs.enabled``springdoc.swagger-ui.enabled` 均为 `false`,此时 `/v3/api-docs``/swagger-ui/index.html` 返回 404。
如果确实需要该文档,更新 `config` 目录下的 `application.yml` 文件显式开启:
```yaml
springdoc:
api-docs:
enabled: true
swagger-ui:
enabled: true
```
开启之后,文档接口仍然被上面的 `resourceRole` 规则收敛在 `admin` 角色。请先以管理员身份登录 HertzBeat Web 应用,再打开 `/swagger-ui/index.html`;Swagger UI 会在同源的文档请求与 try-it-out 请求中附带 HertzBeat 保存的令牌,页面不会再要求任何输入。
没有该会话时文档同样不会泄露,但页面并非静默失败:`/swagger-ui/index.html` 是静态文件,仍然可以打开,而它请求 `/v3/api-docs/swagger-config` 会得到 `401``WWW-Authenticate: Digest` 挑战,浏览器因此弹出用户名密码框。在该弹框中输入管理员账号可以正常加载文档;没有 `admin` 角色的账号则会收到 `403`
也可以直接携带管理员令牌获取文档:
```shell
curl -H "Authorization: Bearer $YOUR_ADMIN_TOKEN" http://localhost:1157/v3/api-docs
```
> ⚠️ 不要把 OpenAPI 路径放回 `excludedResource`,否则完整接口文档会再次匿名开放。
## 更新安全密钥
> 此密钥为账户安全加密管理的密钥,需要更新为相同长度的你自定义密钥串。
+1
View File
@@ -245,6 +245,7 @@
"help/kafka_client",
"help/pulsar",
"help/nacos",
"help/etcd",
"help/rabbitmq",
"help/rocketmq",
"help/shenyu",
+3 -3
View File
@@ -525,9 +525,9 @@ The following components are provided under the MIT License. See project link fo
The text of each license is also included in licenses/LICENSE-[project].txt.
https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.21 MIT
https://mvnrepository.com/artifact/org.influxdb/influxdb-java/2.23 MIT
+3 -3
View File
@@ -524,9 +524,9 @@ The following components are provided under the MIT License. See project link fo
The text of each license is also included in licenses/LICENSE-[project].txt.
https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.21 MIT
https://mvnrepository.com/artifact/org.influxdb/influxdb-java/2.23 MIT
+3 -3
View File
@@ -396,9 +396,9 @@ The following components are provided under the MIT License. See project link fo
The text of each license is also included in licenses/LICENSE-[project].txt.
https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
https://mvnrepository.com/artifact/org.slf4j/jcl-over-slf4j/2.0.9 MIT
https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/2.0.9 MIT
+1
View File
@@ -165,6 +165,7 @@
<consul-api.version>1.4.5</consul-api.version>
<nacos-client.version>3.1.1</nacos-client.version>
<vesoft-client.version>3.6.0</vesoft-client.version>
<bouncycastle.version>1.85</bouncycastle.version>
<jutf7.version>1.0.0</jutf7.version>
<huawei.sdk.version>3.1.37</huawei.sdk.version>
<huawei.obs.version>3.23.5</huawei.obs.version>
+15
View File
@@ -71,6 +71,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -373,3 +383,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume.
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -74,6 +74,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -109,7 +119,7 @@ spring:
hibernate:
format_sql: true
dialect: org.hibernate.dialect.MySQLDialect
flyway:
enabled: true
clean-disabled: true
@@ -117,7 +127,7 @@ spring:
baseline-version: 1
locations:
- classpath:db/migration/mysql
# Not Require, Please config if you need email notify
mail:
# Attention: this is mail server address.
@@ -138,7 +148,7 @@ common:
queue:
# memory or kafka
type: memory
warehouse:
store:
# store history metrics data, enable only one below
@@ -222,7 +232,7 @@ alerter:
region: AWS_REGION_FOR_END_USER_MESSAGING
twilio:
account-sid: YOUR_ACCOUNT_SID
auth-token: YOUR_AUTH_TOKEN
auth-token: YOUR_AUTH_TOKEN
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
scheduler:
server:
@@ -273,3 +283,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -70,6 +70,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -92,11 +109,18 @@ resourceRole:
- /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest]
- /api/logs/ingest/**===post===[admin,user]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -130,10 +154,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -74,6 +74,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -109,7 +119,7 @@ spring:
hibernate:
format_sql: true
dialect: org.hibernate.dialect.MySQLDialect
flyway:
enabled: true
clean-disabled: true
@@ -117,7 +127,7 @@ spring:
baseline-version: 1
locations:
- classpath:db/migration/mysql
# Not Require, Please config if you need email notify
mail:
# Attention: this is mail server address.
@@ -138,7 +148,7 @@ common:
queue:
# memory or kafka
type: memory
warehouse:
store:
# store history metrics data, enable only one below
@@ -219,7 +229,7 @@ alerter:
region: AWS_REGION_FOR_END_USER_MESSAGING
twilio:
account-sid: YOUR_ACCOUNT_SID
auth-token: YOUR_AUTH_TOKEN
auth-token: YOUR_AUTH_TOKEN
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
scheduler:
server:
@@ -270,3 +280,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume.
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -70,6 +70,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -92,11 +109,18 @@ resourceRole:
- /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest]
- /api/logs/ingest/**===post===[admin,user]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -130,10 +154,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -74,6 +74,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -109,7 +119,7 @@ spring:
hibernate:
format_sql: true
dialect: org.hibernate.dialect.MySQLDialect
flyway:
enabled: true
clean-disabled: true
@@ -117,7 +127,7 @@ spring:
baseline-version: 1
locations:
- classpath:db/migration/mysql
# Not Require, Please config if you need email notify
mail:
# Attention: this is mail server address.
@@ -142,7 +152,7 @@ warehouse:
expire-time: 1h
victoria-metrics:
enabled: true
url: http://victoria-metrics:8428
url: http://victoria-metrics:8428
username: root
password: root
insert:
@@ -222,7 +232,7 @@ alerter:
region: AWS_REGION_FOR_END_USER_MESSAGING
twilio:
account-sid: YOUR_ACCOUNT_SID
auth-token: YOUR_AUTH_TOKEN
auth-token: YOUR_AUTH_TOKEN
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
scheduler:
server:
@@ -273,3 +283,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume.
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -70,6 +70,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -92,11 +109,18 @@ resourceRole:
- /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest]
- /api/logs/ingest/**===post===[admin,user]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -130,10 +154,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -74,6 +74,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -108,7 +118,7 @@ spring:
hibernate:
format_sql: true
dialect: org.hibernate.dialect.PostgreSQLDialect
flyway:
enabled: true
clean-disabled: true
@@ -116,7 +126,7 @@ spring:
baseline-version: 1
locations:
- classpath:db/migration/postgresql
# Not Require, Please config if you need email notify
mail:
# Attention: this is mail server address.
@@ -219,7 +229,7 @@ alerter:
region: AWS_REGION_FOR_END_USER_MESSAGING
twilio:
account-sid: YOUR_ACCOUNT_SID
auth-token: YOUR_AUTH_TOKEN
auth-token: YOUR_AUTH_TOKEN
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
scheduler:
server:
@@ -270,3 +280,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume.
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -70,6 +70,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -96,11 +113,18 @@ resourceRole:
- /api/ingestion/otlp/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -137,10 +161,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copy this file to .env and set your own strong password before running docker compose up
POSTGRES_USER=root
POSTGRES_PASSWORD=
@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
.env
@@ -74,6 +74,16 @@ management:
export:
enabled: true
# The generated OpenAPI document is a map of every route, HTTP method, parameter
# and model, so it is not served by default. A deployment that wants the Swagger
# UI opts in by turning both switches on; the document endpoints stay scoped to
# the admin role in sureness.yml either way.
springdoc:
api-docs:
enabled: false
swagger-ui:
enabled: false
sureness:
container: jakarta_servlet
auths:
@@ -108,7 +118,7 @@ spring:
hibernate:
format_sql: true
dialect: org.hibernate.dialect.PostgreSQLDialect
flyway:
enabled: true
clean-disabled: true
@@ -116,7 +126,7 @@ spring:
baseline-version: 1
locations:
- classpath:db/migration/postgresql
# Not Require, Please config if you need email notify
mail:
# Attention: this is mail server address.
@@ -221,7 +231,7 @@ alerter:
region: AWS_REGION_FOR_END_USER_MESSAGING
twilio:
account-sid: YOUR_ACCOUNT_SID
auth-token: YOUR_AUTH_TOKEN
auth-token: YOUR_AUTH_TOKEN
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
scheduler:
server:
@@ -272,3 +282,8 @@ hertzbeat:
concurrency-limit: 256
reject-when-limit-reached: true
task-termination-timeout: 5000
# Bounds on what a single request to the push gateway may consume.
push:
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -70,6 +70,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -92,11 +109,18 @@ resourceRole:
- /api/ai/**===delete===[admin]
- /api/logs/sse/**===get===[admin,user,guest]
- /api/logs/ingest/**===post===[admin,user]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -130,10 +154,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
@@ -34,8 +34,8 @@ services:
ports:
- '15432:5432'
environment:
POSTGRES_USER: root
POSTGRES_PASSWORD: 123456
POSTGRES_USER: ${POSTGRES_USER:-root}
POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD (e.g., in .env)}"
TZ: Asia/Shanghai
PGDATA: /var/lib/postgresql/data/pgdata
volumes:
@@ -73,6 +73,8 @@ services:
HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto
TZ: Asia/Shanghai
LANG: zh_CN.UTF-8
SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER:-root}
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD in your .env file}
depends_on:
postgres:
condition: service_healthy
+26 -6
View File
@@ -16,7 +16,7 @@
## -- sureness.yml account source -- ##
# config the resource restful api that need auth protection, base rbac
# rule: api===method===role
# rule: api===method===role
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
resourceRole:
@@ -70,6 +70,23 @@ resourceRole:
- /api/config/**===post===[admin]
- /api/config/**===put===[admin]
- /api/config/**===delete===[admin]
# queue depth of the hertzbeat process itself, operational data
- /api/metrics===get===[admin]
# per account metric favourites rendered on the monitor pages
- /api/metrics/**===get===[admin,user,guest]
- /api/metrics/**===post===[admin,user,guest]
- /api/metrics/**===delete===[admin,user,guest]
- /api/label/**===get===[admin,user,guest]
- /api/label/**===post===[admin,user]
- /api/label/**===put===[admin,user]
- /api/label/**===delete===[admin]
# the storage availability probe is read by every monitor page, while the query
# route forwards a raw promql expression straight to the time series database
- /api/warehouse/**===get===[admin,user,guest]
- /api/warehouse/query===post===[admin]
- /api/logs/otlp/**===post===[admin,user]
- /api/logs===delete===[admin]
- /api/v2/alerts===post===[admin,user]
- /api/status/page/**===get===[admin,user,guest]
- /api/status/page/**===post===[admin,user]
- /api/status/page/**===put===[admin,user]
@@ -96,11 +113,18 @@ resourceRole:
- /api/ingestion/otlp/**===get===[admin,user,guest]
- /api/logs/**===get===[admin,user,guest]
- /api/traces/**===get===[admin,user,guest]
# The OpenAPI document is a map of every route, parameter and model, so it is
# scoped like any other administrative resource instead of being anonymous
- /v3/api-docs/**===get===[admin]
- /v3/api-docs.yaml===get===[admin]
- /v3/api-docs.yaml/**===get===[admin]
- /v2/api-docs/**===get===[admin]
- /swagger-resources/**===get===[admin]
# spring boot actuator exposes jvm, http and datasource internals for scraping
- /actuator/**===get===[admin]
# config the resource restful api that need bypass auth protection
# rule: api===method
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
excludedResource:
- /api/alert/sse/**===*
@@ -137,10 +161,6 @@ excludedResource:
- /**/*.json===get
- /**/*.woff===get
- /**/*.eot===get
# swagger ui resource
- /swagger-resources/**===get
- /v2/api-docs===get
- /v3/api-docs===get
# h2 database
- /h2-console/**===*
+1 -2
View File
@@ -18,8 +18,7 @@ module.exports = function (config) {
// the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html
// for example, you can disable the random execution with `random: false`
// or set a specific seed with `seed: 4321`
},
clearContext: false // leave Jasmine Spec Runner output visible in browser
}
},
jasmineHtmlReporter: {
suppressAll: true // removes the duplicated traces
+2 -2
View File
@@ -23,12 +23,12 @@
"start": "ng serve --proxy-config proxy.conf.json",
"build": "npm run ng-high-memory build",
"watch": "ng build --watch --configuration development",
"test": "ng test",
"test": "ng test --watch=false --browsers=ChromeHeadless",
"ng-high-memory": "node --max_old_space_size=8000 ./node_modules/@angular/cli/bin/ng",
"hmr": "ng s -o --hmr",
"analyze": "npm run ng-high-memory build -- --source-map",
"analyze:view": "source-map-explorer dist/**/*.js",
"test-coverage": "ng test --code-coverage --watch=false",
"test-coverage": "ng test --code-coverage --watch=false --browsers=ChromeHeadless",
"color-less": "ng-alain-plugin-theme -t=colorLess",
"theme": "ng-alain-plugin-theme -t=themeCss",
"icon": "ng g ng-alain:plugin icon",
@@ -71,7 +71,7 @@ describe('Service: I18n', () => {
it('should be use default language when the browser language is not in the list', () => {
spyOnProperty(navigator, 'languages').and.returnValue(['es-419']);
genModule();
expect(srv.defaultLang).toBe('zh-CN');
expect(srv.defaultLang).toBe('en-US');
});
it('should be trigger notify when changed language', () => {
@@ -18,7 +18,7 @@
*/
import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing';
import { Component, NgZone } from '@angular/core';
import { Component } from '@angular/core';
import { ComponentFixture, TestBed, fakeAsync, tick } from '@angular/core/testing';
import { By } from '@angular/platform-browser';
import { I18NService } from '@core';
@@ -45,6 +45,7 @@ describe('SettingDrawerI18nDirective', () => {
let i18nService: jasmine.SpyObj<I18NService>;
let httpMock: HttpTestingController;
let mockTranslations: { [key: string]: string };
const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR'];
const mockI18nData = {
'zh-CN': {
@@ -102,7 +103,7 @@ describe('SettingDrawerI18nDirective', () => {
await TestBed.configureTestingModule({
imports: [HttpClientTestingModule],
declarations: [TestComponent, SettingDrawerI18nDirective],
providers: [{ provide: ALAIN_I18N_TOKEN, useValue: i18nServiceSpy }, NgZone]
providers: [{ provide: ALAIN_I18N_TOKEN, useValue: i18nServiceSpy }]
}).compileComponents();
fixture = TestBed.createComponent(TestComponent);
@@ -120,36 +121,33 @@ describe('SettingDrawerI18nDirective', () => {
httpMock.verify();
});
it('should create', () => {
function loadMappings(): void {
fixture.detectChanges();
languages.forEach(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`).flush(mockI18nData[lang as keyof typeof mockI18nData]));
}
function destroyFixture(): void {
fixture.destroy();
tick(2000);
}
it('should create', fakeAsync(() => {
loadMappings();
expect(directive).toBeTruthy();
});
destroyFixture();
}));
it('should load mappings from i18n files', fakeAsync(() => {
fixture.detectChanges();
const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR'];
const requests = languages.map(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`));
languages.forEach((lang, index) => {
requests[index].flush(mockI18nData[lang as keyof typeof mockI18nData]);
});
loadMappings();
tick(100);
fixture.detectChanges();
expect(i18nService.fanyi).toHaveBeenCalled();
destroyFixture();
}));
it('should replace Chinese text with translations', fakeAsync(() => {
fixture.detectChanges();
const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR'];
const requests = languages.map(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`));
languages.forEach((lang, index) => {
requests[index].flush(mockI18nData[lang as keyof typeof mockI18nData]);
});
loadMappings();
tick(2000);
fixture.detectChanges();
tick(100);
@@ -161,5 +159,6 @@ describe('SettingDrawerI18nDirective', () => {
expect(themeColorDiv.textContent).toContain('Theme Color');
}
}
destroyFixture();
}));
});
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertCenterComponent } from './alert-center.component';
@@ -26,9 +27,7 @@ describe('AlertCenterComponent', () => {
let fixture: ComponentFixture<AlertCenterComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [AlertCenterComponent]
}).compileComponents();
await configureShallowTest(AlertCenterComponent).compileComponents();
});
beforeEach(() => {
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertGroupConvergeComponent } from './alert-group-converge.component';
@@ -26,9 +27,7 @@ describe('AlertConvergeComponent', () => {
let fixture: ComponentFixture<AlertGroupConvergeComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [AlertGroupConvergeComponent]
}).compileComponents();
await configureShallowTest(AlertGroupConvergeComponent).compileComponents();
fixture = TestBed.createComponent(AlertGroupConvergeComponent);
component = fixture.componentInstance;
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertInhibitComponent } from './alert-inhibit.component';
@@ -26,9 +27,7 @@ describe('AlertInhibitComponent', () => {
let fixture: ComponentFixture<AlertInhibitComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AlertInhibitComponent]
}).compileComponents();
await configureShallowTest(AlertInhibitComponent).compileComponents();
fixture = TestBed.createComponent(AlertInhibitComponent);
component = fixture.componentInstance;
@@ -18,6 +18,8 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureStandaloneTest } from '@testing';
import { MarkdownModule } from 'ngx-markdown';
import { AlertIntegrationComponent } from './alert-integration.component';
@@ -26,9 +28,7 @@ describe('AlertIntegrationComponent', () => {
let fixture: ComponentFixture<AlertIntegrationComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AlertIntegrationComponent]
}).compileComponents();
await configureStandaloneTest(AlertIntegrationComponent, [MarkdownModule.forRoot()]).compileComponents();
fixture = TestBed.createComponent(AlertIntegrationComponent);
component = fixture.componentInstance;
@@ -80,6 +80,11 @@ export class AlertIntegrationComponent implements OnInit {
name: this.i18nSvc.fanyi('alert.integration.source.tencent'),
icon: 'assets/img/integration/tencent.svg'
},
{
id: 'alibabacloud-cms',
name: this.i18nSvc.fanyi('alert.integration.source.alibabacloud-cms'),
icon: 'assets/img/integration/alibabacloud.svg'
},
{
id: 'alibabacloud-sls',
name: this.i18nSvc.fanyi('alert.integration.source.alibabacloud-sls'),
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertNoticeReceiverComponent } from './alert-notice-receiver.component';
@@ -26,9 +27,7 @@ describe('AlertNoticeReceiverComponent', () => {
let fixture: ComponentFixture<AlertNoticeReceiverComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AlertNoticeReceiverComponent]
}).compileComponents();
await configureShallowTest(AlertNoticeReceiverComponent).compileComponents();
fixture = TestBed.createComponent(AlertNoticeReceiverComponent);
component = fixture.componentInstance;
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertNoticeRuleComponent } from './alert-notice-rule.component';
@@ -26,9 +27,7 @@ describe('AlertNoticeRuleComponent', () => {
let fixture: ComponentFixture<AlertNoticeRuleComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AlertNoticeRuleComponent]
}).compileComponents();
await configureShallowTest(AlertNoticeRuleComponent).compileComponents();
fixture = TestBed.createComponent(AlertNoticeRuleComponent);
component = fixture.componentInstance;
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertNoticeTemplateComponent } from './alert-notice-template.component';
@@ -26,9 +27,7 @@ describe('AlertNoticeTemplateComponent', () => {
let fixture: ComponentFixture<AlertNoticeTemplateComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
imports: [AlertNoticeTemplateComponent]
}).compileComponents();
await configureShallowTest(AlertNoticeTemplateComponent).compileComponents();
fixture = TestBed.createComponent(AlertNoticeTemplateComponent);
component = fixture.componentInstance;
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertNoticeComponent } from './alert-notice.component';
@@ -26,9 +27,7 @@ describe('AlertNoticeComponent', () => {
let fixture: ComponentFixture<AlertNoticeComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [AlertNoticeComponent]
}).compileComponents();
await configureShallowTest(AlertNoticeComponent).compileComponents();
});
beforeEach(() => {
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertSettingComponent } from './alert-setting.component';
@@ -26,9 +27,7 @@ describe('AlertSettingComponent', () => {
let fixture: ComponentFixture<AlertSettingComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [AlertSettingComponent]
}).compileComponents();
await configureShallowTest(AlertSettingComponent).compileComponents();
});
beforeEach(() => {
@@ -18,6 +18,7 @@
*/
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { configureShallowTest } from '@testing';
import { AlertSilenceComponent } from './alert-silence.component';
@@ -26,9 +27,7 @@ describe('AlertSilenceComponent', () => {
let fixture: ComponentFixture<AlertSilenceComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [AlertSilenceComponent]
}).compileComponents();
await configureShallowTest(AlertSilenceComponent).compileComponents();
fixture = TestBed.createComponent(AlertSilenceComponent);
component = fixture.componentInstance;

Some files were not shown because too many files have changed in this diff Show More