mirror of
https://github.com/dromara/hertzbeat.git
synced 2026-09-17 09:40:58 +00:00
Merge branch 'master' into fix/008-questdb-sql-injection
This commit is contained in:
@@ -37,6 +37,7 @@ concurrency:
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
@@ -53,3 +54,6 @@ jobs:
|
||||
- name: EsLint Test
|
||||
working-directory: web-app
|
||||
run: pnpm lint:ts
|
||||
- name: Unit Test
|
||||
working-directory: web-app
|
||||
run: pnpm test
|
||||
|
||||
+26
-14
@@ -78,24 +78,32 @@ public class ConversationServiceImpl implements ConversationService {
|
||||
.build());
|
||||
}
|
||||
|
||||
log.info("Starting streaming conversation: {}", conversationId);
|
||||
ChatConversation conversation = conversationDao.findById(conversationId)
|
||||
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
|
||||
ChatConversation conversation;
|
||||
if (conversationId == null) {
|
||||
// The API contract makes conversationId optional, so create a conversation for the first message.
|
||||
conversation = new ChatConversation();
|
||||
conversation.setTitle(buildConversationTitle(message));
|
||||
conversation = conversationDao.save(conversation);
|
||||
} else {
|
||||
conversation = conversationDao.findById(conversationId)
|
||||
.orElseThrow(() -> new IllegalArgumentException("Conversation not found: " + conversationId));
|
||||
}
|
||||
Long currentConversationId = conversation.getId();
|
||||
log.info("Starting streaming conversation: {}", currentConversationId);
|
||||
|
||||
// Manually load messages for conversation history
|
||||
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(conversationId);
|
||||
List<ChatMessage> messages = messageDao.findByConversationIdOrderByGmtCreateAsc(currentConversationId);
|
||||
conversation.setMessages(messages);
|
||||
|
||||
if (conversation.getTitle().startsWith("conversation")) {
|
||||
// Auto-generate title from first user message
|
||||
String title = message.length() > 30 ? message.substring(0, 27) + "..." : message;
|
||||
conversation.setTitle(title);
|
||||
conversation.setTitle(buildConversationTitle(message));
|
||||
conversationDao.save(conversation);
|
||||
}
|
||||
|
||||
// Add user message to conversation
|
||||
ChatMessage chatMessage = ChatMessage.builder()
|
||||
.conversationId(conversationId)
|
||||
.conversationId(currentConversationId)
|
||||
.content(message)
|
||||
.role("user")
|
||||
.build();
|
||||
@@ -103,7 +111,7 @@ public class ConversationServiceImpl implements ConversationService {
|
||||
|
||||
ChatRequestContext context = ChatRequestContext.builder()
|
||||
.message(message)
|
||||
.conversationId(conversationId)
|
||||
.conversationId(currentConversationId)
|
||||
.conversationHistory(messages)
|
||||
.build();
|
||||
|
||||
@@ -116,7 +124,7 @@ public class ConversationServiceImpl implements ConversationService {
|
||||
.map(chunk -> {
|
||||
fullResponse.append(chunk);
|
||||
ChatResponseChunk responseChunk = ChatResponseChunk.builder()
|
||||
.conversationId(conversationId)
|
||||
.conversationId(currentConversationId)
|
||||
.userMessageId(finalChatMessage.getId())
|
||||
.response(chunk)
|
||||
.build();
|
||||
@@ -128,13 +136,13 @@ public class ConversationServiceImpl implements ConversationService {
|
||||
.concatWith(Flux.defer(() -> {
|
||||
// Add the complete AI response to conversation
|
||||
ChatMessage assistantMessage = ChatMessage.builder()
|
||||
.conversationId(conversationId)
|
||||
.conversationId(currentConversationId)
|
||||
.content(fullResponse.toString())
|
||||
.role("assistant")
|
||||
.build();
|
||||
assistantMessage = messageDao.save(assistantMessage);
|
||||
ChatResponseChunk finalResponse = ChatResponseChunk.builder()
|
||||
.conversationId(conversationId)
|
||||
.conversationId(currentConversationId)
|
||||
.response("")
|
||||
.assistantMessageId(assistantMessage.getId())
|
||||
.build();
|
||||
@@ -143,12 +151,12 @@ public class ConversationServiceImpl implements ConversationService {
|
||||
.event("complete")
|
||||
.build());
|
||||
}))
|
||||
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", conversationId))
|
||||
.doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", conversationId,
|
||||
.doOnComplete(() -> log.info("Streaming completed for conversation: {}", currentConversationId))
|
||||
.doOnError(error -> log.error("Error in streaming chat for conversation {}: {}", currentConversationId,
|
||||
error.getMessage(), error))
|
||||
.onErrorResume(error -> {
|
||||
ChatResponseChunk errorResponse = ChatResponseChunk.builder()
|
||||
.conversationId(conversationId)
|
||||
.conversationId(currentConversationId)
|
||||
.response("An error occurred: " + error.getMessage())
|
||||
.userMessageId(finalChatMessage.getId())
|
||||
.build();
|
||||
@@ -165,6 +173,10 @@ public class ConversationServiceImpl implements ConversationService {
|
||||
return conversationDao.save(conversation);
|
||||
}
|
||||
|
||||
private String buildConversationTitle(String message) {
|
||||
return message.length() > 30 ? message.substring(0, 27) + "..." : message;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatConversation getConversation(Long conversationId) {
|
||||
if (conversationId == null) {
|
||||
|
||||
+70
-5
@@ -19,17 +19,23 @@ package org.apache.hertzbeat.ai.tools.impl;
|
||||
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.ai.service.SopScheduleService;
|
||||
import org.apache.hertzbeat.ai.sop.model.SopDefinition;
|
||||
import org.apache.hertzbeat.ai.sop.model.SopParameter;
|
||||
import org.apache.hertzbeat.ai.sop.registry.SkillRegistry;
|
||||
import org.apache.hertzbeat.ai.utils.SopMessageUtil;
|
||||
import org.apache.hertzbeat.ai.tools.ScheduleTools;
|
||||
import org.apache.hertzbeat.common.entity.ai.SopSchedule;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.stereotype.Service;
|
||||
import tools.jackson.core.type.TypeReference;
|
||||
|
||||
/**
|
||||
* Implementation of ScheduleTools for AI-driven schedule management.
|
||||
@@ -78,18 +84,22 @@ public class ScheduleToolsImpl implements ScheduleTools {
|
||||
@Tool(name = "createScheduleWithConversation",
|
||||
description = "Create a scheduled task for a specific conversation. "
|
||||
+ "Use the conversationId from the system context. "
|
||||
+ "Pass skill parameters as a JSON object when the skill requires inputs. "
|
||||
+ "The cron expression should be in 6-digit Spring format.")
|
||||
public String createScheduleWithConversation(
|
||||
@ToolParam(description = "Conversation ID from the system context", required = true) Long conversationId,
|
||||
@ToolParam(description = "Name of the skill to schedule (e.g., 'daily_inspection')", required = true) String skillName,
|
||||
@ToolParam(description = "Cron expression in Spring format (e.g., '0 0 9 * * ?')", required = true) String cronExpression,
|
||||
@ToolParam(description = "Description of the schedule", required = false) String description) {
|
||||
@ToolParam(description = "Description of the schedule", required = false) String description,
|
||||
@ToolParam(description = "Skill parameters as a JSON object (e.g., '{\"monitorId\":123}')",
|
||||
required = false) String paramsJson) {
|
||||
|
||||
log.info("AI creating schedule: conversationId={}, skill={}, cron={}, desc={}",
|
||||
conversationId, skillName, cronExpression, description);
|
||||
|
||||
// Validate skill exists
|
||||
if (skillRegistry.getSkill(skillName) == null) {
|
||||
SopDefinition skill = skillRegistry.getSkill(skillName);
|
||||
if (skill == null) {
|
||||
String available = String.join(", ",
|
||||
skillRegistry.getAllSkills().stream()
|
||||
.map(s -> s.getName())
|
||||
@@ -103,11 +113,16 @@ public class ScheduleToolsImpl implements ScheduleTools {
|
||||
}
|
||||
|
||||
try {
|
||||
// Check for duplicate schedule (same skill + cron expression)
|
||||
Map<String, Object> params = parseSkillParams(paramsJson);
|
||||
validateRequiredParameters(skill, params);
|
||||
String serializedParams = params.isEmpty() ? null : JsonUtil.toJson(params);
|
||||
|
||||
// Parameters are part of a schedule's identity so the same skill and cron can target different inputs.
|
||||
List<SopSchedule> existing = scheduleService.getSchedulesByConversation(conversationId);
|
||||
boolean duplicate = existing.stream()
|
||||
.anyMatch(s -> s.getSopName().equals(skillName)
|
||||
&& s.getCronExpression().equals(cronExpression));
|
||||
.anyMatch(schedule -> Objects.equals(schedule.getSopName(), skillName)
|
||||
&& Objects.equals(schedule.getCronExpression(), cronExpression)
|
||||
&& hasSameParams(schedule.getSopParams(), params));
|
||||
if (duplicate) {
|
||||
return SopMessageUtil.getMessage("schedule.create.duplicate",
|
||||
new Object[]{skillName, cronExpression}, null)
|
||||
@@ -118,6 +133,7 @@ public class ScheduleToolsImpl implements ScheduleTools {
|
||||
schedule.setConversationId(conversationId);
|
||||
schedule.setSopName(skillName);
|
||||
schedule.setCronExpression(cronExpression);
|
||||
schedule.setSopParams(serializedParams);
|
||||
schedule.setEnabled(true);
|
||||
|
||||
SopSchedule created = scheduleService.createSchedule(schedule);
|
||||
@@ -150,6 +166,55 @@ public class ScheduleToolsImpl implements ScheduleTools {
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, Object> parseSkillParams(String paramsJson) {
|
||||
if (paramsJson == null || paramsJson.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
Map<String, Object> params;
|
||||
try {
|
||||
params = JsonUtil.fromJson(paramsJson, new TypeReference<>() {});
|
||||
} catch (RuntimeException e) {
|
||||
throw new IllegalArgumentException("Skill parameters must be a valid JSON object", e);
|
||||
}
|
||||
if (params == null) {
|
||||
throw new IllegalArgumentException("Skill parameters must be a valid JSON object");
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
private void validateRequiredParameters(SopDefinition skill, Map<String, Object> params) {
|
||||
if (skill.getParameters() == null) {
|
||||
return;
|
||||
}
|
||||
for (SopParameter parameter : skill.getParameters()) {
|
||||
Object value = params.get(parameter.getName());
|
||||
if (isMissing(value)) {
|
||||
value = parameter.getDefaultValue();
|
||||
}
|
||||
if (parameter.isRequired() && isMissing(value)) {
|
||||
throw new IllegalArgumentException(
|
||||
"Required skill parameter is missing: " + parameter.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isMissing(Object value) {
|
||||
return value == null || value instanceof String text && text.isBlank();
|
||||
}
|
||||
|
||||
private boolean hasSameParams(String existingJson, Map<String, Object> params) {
|
||||
if (existingJson == null || existingJson.isBlank()) {
|
||||
return params.isEmpty();
|
||||
}
|
||||
try {
|
||||
Map<String, Object> existingParams = JsonUtil.fromJson(existingJson, new TypeReference<>() {});
|
||||
return Objects.equals(existingParams, params);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("Failed to parse parameters of an existing schedule", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@Tool(name = "listSchedulesForConversation",
|
||||
description = "List all scheduled tasks for a specific conversation. "
|
||||
|
||||
+42
@@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.inOrder;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.usthe.sureness.subject.SubjectSum;
|
||||
@@ -123,6 +124,47 @@ class ConversationServiceImplTest {
|
||||
assertEquals(subject, contextCaptor.getValue().getSubject());
|
||||
}
|
||||
|
||||
/**
|
||||
* The service should create a conversation and return its ID when the client omits the optional conversation ID.
|
||||
*/
|
||||
@Test
|
||||
void streamChatShouldCreateConversationWhenConversationIdIsMissing() {
|
||||
AtomicLong messageId = new AtomicLong(20L);
|
||||
when(chatClientProviderService.isConfigured()).thenReturn(true);
|
||||
when(conversationDao.save(any(ChatConversation.class))).thenAnswer(invocation -> {
|
||||
ChatConversation savedConversation = invocation.getArgument(0);
|
||||
savedConversation.setId(CONVERSATION_ID);
|
||||
return savedConversation;
|
||||
});
|
||||
when(messageDao.findByConversationIdOrderByGmtCreateAsc(CONVERSATION_ID)).thenReturn(List.of());
|
||||
when(messageDao.save(any(ChatMessage.class))).thenAnswer(invocation -> {
|
||||
ChatMessage savedMessage = invocation.getArgument(0);
|
||||
savedMessage.setId(messageId.getAndIncrement());
|
||||
return savedMessage;
|
||||
});
|
||||
when(chatClientProviderService.streamChat(any(ChatRequestContext.class)))
|
||||
.thenReturn(Flux.just("本轮回答"));
|
||||
|
||||
List<ServerSentEvent<ChatResponseChunk>> events = conversationService
|
||||
.streamChat("本轮问题", null)
|
||||
.collectList()
|
||||
.block();
|
||||
|
||||
assertNotNull(events);
|
||||
assertEquals(2, events.size());
|
||||
assertEquals(CONVERSATION_ID, events.get(0).data().getConversationId());
|
||||
assertEquals(CONVERSATION_ID, events.get(1).data().getConversationId());
|
||||
|
||||
ArgumentCaptor<ChatRequestContext> contextCaptor = ArgumentCaptor.forClass(ChatRequestContext.class);
|
||||
verify(chatClientProviderService).streamChat(contextCaptor.capture());
|
||||
assertEquals(CONVERSATION_ID, contextCaptor.getValue().getConversationId());
|
||||
assertEquals(List.of(), contextCaptor.getValue().getConversationHistory());
|
||||
ArgumentCaptor<ChatConversation> conversationCaptor = ArgumentCaptor.forClass(ChatConversation.class);
|
||||
verify(conversationDao).save(conversationCaptor.capture());
|
||||
assertEquals("本轮问题", conversationCaptor.getValue().getTitle());
|
||||
verifyNoMoreInteractions(conversationDao);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deleting a conversation must remove its schedules before they can push more messages.
|
||||
*/
|
||||
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.ai.tools.impl;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.ai.service.SopScheduleService;
|
||||
import org.apache.hertzbeat.ai.sop.model.SopDefinition;
|
||||
import org.apache.hertzbeat.ai.sop.model.SopParameter;
|
||||
import org.apache.hertzbeat.ai.sop.registry.SkillRegistry;
|
||||
import org.apache.hertzbeat.common.entity.ai.SopSchedule;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
/**
|
||||
* Verifies that AI-created SOP schedules validate and persist skill parameters.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ScheduleToolsImplTest {
|
||||
|
||||
private static final String CRON = "0 0 9 * * ?";
|
||||
|
||||
@Mock
|
||||
private SopScheduleService scheduleService;
|
||||
|
||||
@Mock
|
||||
private SkillRegistry skillRegistry;
|
||||
|
||||
private ScheduleToolsImpl scheduleTools;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
scheduleTools = new ScheduleToolsImpl(scheduleService, skillRegistry);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createScheduleShouldPersistSkillParameters() {
|
||||
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
|
||||
when(scheduleService.getSchedulesByConversation(7L)).thenReturn(List.of());
|
||||
when(scheduleService.createSchedule(any())).thenAnswer(invocation -> {
|
||||
SopSchedule schedule = invocation.getArgument(0);
|
||||
schedule.setId(9L);
|
||||
return schedule;
|
||||
});
|
||||
|
||||
String result = scheduleTools.createScheduleWithConversation(
|
||||
7L, "diagnosis", CRON, "daily diagnosis", "{\"monitorId\":42}");
|
||||
|
||||
ArgumentCaptor<SopSchedule> captor = ArgumentCaptor.forClass(SopSchedule.class);
|
||||
verify(scheduleService).createSchedule(captor.capture());
|
||||
assertEquals("{\"monitorId\":42}", captor.getValue().getSopParams());
|
||||
assertTrue(result.contains("9"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void createScheduleShouldAllowDifferentParametersAtTheSameTime() {
|
||||
SopSchedule existing = SopSchedule.builder()
|
||||
.sopName("diagnosis")
|
||||
.cronExpression(CRON)
|
||||
.sopParams("{\"monitorId\":41}")
|
||||
.build();
|
||||
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
|
||||
when(scheduleService.getSchedulesByConversation(7L)).thenReturn(List.of(existing));
|
||||
when(scheduleService.createSchedule(any())).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
scheduleTools.createScheduleWithConversation(
|
||||
7L, "diagnosis", CRON, null, "{\"monitorId\":42}");
|
||||
|
||||
verify(scheduleService).createSchedule(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void createScheduleShouldRejectMissingRequiredParameter() {
|
||||
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
|
||||
|
||||
String result = scheduleTools.createScheduleWithConversation(
|
||||
7L, "diagnosis", CRON, null, "{}");
|
||||
|
||||
assertTrue(result.contains("monitorId"));
|
||||
verify(scheduleService, never()).createSchedule(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void createScheduleShouldRejectInvalidParameterJson() {
|
||||
when(skillRegistry.getSkill("diagnosis")).thenReturn(parameterizedSkill());
|
||||
|
||||
String result = scheduleTools.createScheduleWithConversation(
|
||||
7L, "diagnosis", CRON, null, "not-json");
|
||||
|
||||
assertTrue(result.contains("valid JSON object"));
|
||||
verify(scheduleService, never()).createSchedule(any());
|
||||
}
|
||||
|
||||
private SopDefinition parameterizedSkill() {
|
||||
SopParameter monitorId = SopParameter.builder()
|
||||
.name("monitorId")
|
||||
.required(true)
|
||||
.build();
|
||||
return SopDefinition.builder()
|
||||
.name("diagnosis")
|
||||
.parameters(List.of(monitorId))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
+5
-11
@@ -251,20 +251,14 @@ public class MetricsRealTimeAlertCalculator {
|
||||
}
|
||||
final int fieldType = field.getType();
|
||||
|
||||
// strict jexl aborts the whole rule on undefined variables,
|
||||
// so define every field even when its value is empty or unparseable
|
||||
if (fieldType == CommonConstants.TYPE_NUMBER) {
|
||||
final Double doubleValue;
|
||||
if ((doubleValue = CommonUtil.parseStrDouble(valueStr)) != null) {
|
||||
fieldValueMap.put(field.getName(), doubleValue);
|
||||
}
|
||||
fieldValueMap.put(field.getName(), CommonUtil.parseStrDouble(valueStr));
|
||||
} else if (fieldType == CommonConstants.TYPE_TIME) {
|
||||
final Integer integerValue;
|
||||
if ((integerValue = CommonUtil.parseStrInteger(valueStr)) != null) {
|
||||
fieldValueMap.put(field.getName(), integerValue);
|
||||
}
|
||||
fieldValueMap.put(field.getName(), CommonUtil.parseStrInteger(valueStr));
|
||||
} else {
|
||||
if (StringUtils.isNotEmpty(valueStr)) {
|
||||
fieldValueMap.put(field.getName(), valueStr);
|
||||
}
|
||||
fieldValueMap.put(field.getName(), valueStr);
|
||||
}
|
||||
|
||||
if (field.getLabel()) {
|
||||
|
||||
@@ -51,4 +51,6 @@ public class AlertDefineDTO {
|
||||
private String template;
|
||||
@Excel(name = "Enable")
|
||||
private Boolean enable;
|
||||
@Excel(name = "Datasource")
|
||||
private String datasource;
|
||||
}
|
||||
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.alert.dto;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||
import java.util.Map;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* Alibaba Cloud Monitor 2.0 webhook alert entity.
|
||||
*
|
||||
* @see <a href="https://help.aliyun.com/zh/cms/cloudmonitor-2-0/notification-object">
|
||||
* Alibaba Cloud Monitor webhook payload fields</a>
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class AlibabaCloudCmsExternAlert {
|
||||
|
||||
private String specversion;
|
||||
|
||||
private String id;
|
||||
|
||||
private String type;
|
||||
|
||||
private String subtype;
|
||||
|
||||
private String source;
|
||||
|
||||
private String sourcetype;
|
||||
|
||||
private String time;
|
||||
|
||||
private Long timestamp;
|
||||
|
||||
private String subject;
|
||||
|
||||
private String datacontenttype;
|
||||
|
||||
private String severity;
|
||||
|
||||
private String status;
|
||||
|
||||
private String userId;
|
||||
|
||||
private String ruleId;
|
||||
|
||||
private String workspace;
|
||||
|
||||
private String traceId;
|
||||
|
||||
private String alertMessage;
|
||||
|
||||
private String alertEntityId;
|
||||
|
||||
private Resource resource;
|
||||
|
||||
private Map<String, Object> labels;
|
||||
|
||||
private Map<String, Object> annotations;
|
||||
|
||||
private AlertData data;
|
||||
|
||||
private Map<String, Object> alertEntityFields;
|
||||
|
||||
private String ruleUrl;
|
||||
|
||||
private String entityUrl;
|
||||
|
||||
private String alertRuleUrl;
|
||||
|
||||
private String alertHistoryUrl;
|
||||
|
||||
/**
|
||||
* Alert resource.
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class Resource {
|
||||
|
||||
private Entity entity;
|
||||
|
||||
private Map<String, Object> tags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Alert resource entity.
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class Entity {
|
||||
|
||||
private String domain;
|
||||
|
||||
@JsonAlias("entity_type")
|
||||
private String entityType;
|
||||
|
||||
@JsonAlias("entity_id")
|
||||
private String entityId;
|
||||
|
||||
private Map<String, Object> prop;
|
||||
}
|
||||
|
||||
/**
|
||||
* Threshold alert data.
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class AlertData {
|
||||
|
||||
private Object value;
|
||||
|
||||
private Object threshold;
|
||||
|
||||
private String comparisonOperator;
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -155,6 +155,7 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx
|
||||
alertDefineDTO.setAnnotations(JsonUtil.fromJson(getCellValueAsString(row.getCell(6)), typeReference));
|
||||
alertDefineDTO.setTemplate(getCellValueAsString(row.getCell(7)));
|
||||
alertDefineDTO.setEnable(getCellValueAsBoolean(row.getCell(8)));
|
||||
alertDefineDTO.setDatasource(getCellValueAsString(row.getCell(9)));
|
||||
return alertDefineDTO;
|
||||
}
|
||||
|
||||
@@ -186,7 +187,7 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx
|
||||
CellStyle cellStyle = workbook.createCellStyle();
|
||||
cellStyle.setAlignment(HorizontalAlignment.CENTER);
|
||||
// set header
|
||||
String[] headers = {"Name", "Type", "Expr", "Period", "Times", "Labels", "Annotations", "Template", "Enable"};
|
||||
String[] headers = {"Name", "Type", "Expr", "Period", "Times", "Labels", "Annotations", "Template", "Enable", "Datasource"};
|
||||
Row headerRow = sheet.createRow(0);
|
||||
for (int i = 0; i < headers.length; i++) {
|
||||
Cell cell = headerRow.createCell(i);
|
||||
@@ -227,6 +228,9 @@ public class AlertDefineExcelImExportServiceImpl extends AlertDefineAbstractImEx
|
||||
Cell enableCell = row.createCell(8);
|
||||
enableCell.setCellValue(alertDefineDTO.getEnable());
|
||||
enableCell.setCellStyle(cellStyle);
|
||||
Cell datasourceCell = row.createCell(9);
|
||||
datasourceCell.setCellValue(alertDefineDTO.getDatasource());
|
||||
datasourceCell.setCellStyle(cellStyle);
|
||||
}
|
||||
workbook.write(os);
|
||||
os.close();
|
||||
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.alert.service.impl;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.format.DateTimeParseException;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.alert.dto.AlibabaCloudCmsExternAlert;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
|
||||
import org.apache.hertzbeat.alert.service.ExternAlertService;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* Alibaba Cloud Monitor 2.0 external alert service.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class AlibabaCloudCmsExternAlertService implements ExternAlertService {
|
||||
|
||||
private static final String SOURCE = "alibabacloud-cms";
|
||||
|
||||
private final AlarmCommonReduce alarmCommonReduce;
|
||||
|
||||
public AlibabaCloudCmsExternAlertService(AlarmCommonReduce alarmCommonReduce) {
|
||||
this.alarmCommonReduce = alarmCommonReduce;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addExternAlert(String content) {
|
||||
AlibabaCloudCmsExternAlert externAlert = JsonUtil.fromJson(content, AlibabaCloudCmsExternAlert.class);
|
||||
if (externAlert == null || StringUtils.isBlank(externAlert.getStatus())) {
|
||||
log.warn("Failed to parse Alibaba Cloud Monitor external alert content: {}", content);
|
||||
return;
|
||||
}
|
||||
alarmCommonReduce.reduceAndSendAlarm(convert(externAlert));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String supportSource() {
|
||||
return SOURCE;
|
||||
}
|
||||
|
||||
private SingleAlert convert(AlibabaCloudCmsExternAlert externAlert) {
|
||||
boolean resolved = isResolved(externAlert);
|
||||
long eventTime = getEventTime(externAlert);
|
||||
return SingleAlert.builder()
|
||||
.content(getAlertContent(externAlert))
|
||||
.status(resolved ? CommonConstants.ALERT_STATUS_RESOLVED : CommonConstants.ALERT_STATUS_FIRING)
|
||||
.startAt(eventTime)
|
||||
.activeAt(resolved ? null : eventTime)
|
||||
.endAt(resolved ? eventTime : null)
|
||||
.labels(buildLabels(externAlert))
|
||||
.annotations(buildAnnotations(externAlert))
|
||||
.triggerTimes(1)
|
||||
.build();
|
||||
}
|
||||
|
||||
private boolean isResolved(AlibabaCloudCmsExternAlert externAlert) {
|
||||
return "RESOLVED".equalsIgnoreCase(externAlert.getStatus())
|
||||
|| "RECOVERED".equalsIgnoreCase(externAlert.getStatus())
|
||||
|| "NORMAL_RESOLVE".equalsIgnoreCase(externAlert.getSubtype());
|
||||
}
|
||||
|
||||
private long getEventTime(AlibabaCloudCmsExternAlert externAlert) {
|
||||
if (externAlert.getTimestamp() != null && externAlert.getTimestamp() > 0) {
|
||||
return externAlert.getTimestamp();
|
||||
}
|
||||
if (StringUtils.isNotBlank(externAlert.getTime())) {
|
||||
try {
|
||||
return Instant.parse(externAlert.getTime()).toEpochMilli();
|
||||
} catch (DateTimeParseException e) {
|
||||
log.warn("Failed to parse Alibaba Cloud Monitor event time: {}", externAlert.getTime());
|
||||
}
|
||||
}
|
||||
return Instant.now().toEpochMilli();
|
||||
}
|
||||
|
||||
private Map<String, String> buildLabels(AlibabaCloudCmsExternAlert externAlert) {
|
||||
Map<String, String> labels = new HashMap<>(16);
|
||||
putValues(labels, externAlert.getLabels());
|
||||
AlibabaCloudCmsExternAlert.Resource resource = externAlert.getResource();
|
||||
if (resource != null) {
|
||||
putValues(labels, resource.getTags());
|
||||
AlibabaCloudCmsExternAlert.Entity entity = resource.getEntity();
|
||||
if (entity != null) {
|
||||
putIfNotBlank(labels, "resourceDomain", entity.getDomain());
|
||||
putIfNotBlank(labels, "resourceType", entity.getEntityType());
|
||||
putIfNotBlank(labels, "resourceId", entity.getEntityId());
|
||||
}
|
||||
}
|
||||
labels.put("__source__", SOURCE);
|
||||
putIfNotBlank(labels, CommonConstants.LABEL_ALERT_NAME, externAlert.getSubject());
|
||||
putIfNotBlank(labels, CommonConstants.LABEL_ALERT_SEVERITY, convertSeverity(externAlert.getSeverity()));
|
||||
putIfNotBlank(labels, "ruleId", externAlert.getRuleId());
|
||||
putIfNotBlank(labels, "workspace", externAlert.getWorkspace());
|
||||
putIfNotBlank(labels, "alertEntityId", externAlert.getAlertEntityId());
|
||||
putIfNotBlank(labels, "userId", externAlert.getUserId());
|
||||
return labels;
|
||||
}
|
||||
|
||||
private Map<String, String> buildAnnotations(AlibabaCloudCmsExternAlert externAlert) {
|
||||
Map<String, String> annotations = new HashMap<>(16);
|
||||
putValues(annotations, externAlert.getAnnotations());
|
||||
AlibabaCloudCmsExternAlert.Resource resource = externAlert.getResource();
|
||||
if (resource != null && resource.getEntity() != null) {
|
||||
putValues(annotations, resource.getEntity().getProp());
|
||||
}
|
||||
putValues(annotations, externAlert.getAlertEntityFields());
|
||||
AlibabaCloudCmsExternAlert.AlertData data = externAlert.getData();
|
||||
if (data != null) {
|
||||
putValue(annotations, "value", data.getValue());
|
||||
putValue(annotations, "threshold", data.getThreshold());
|
||||
putIfNotBlank(annotations, "comparisonOperator", data.getComparisonOperator());
|
||||
}
|
||||
putIfNotBlank(annotations, "alertMessage", externAlert.getAlertMessage());
|
||||
putIfNotBlank(annotations, "traceId", externAlert.getTraceId());
|
||||
putIfNotBlank(annotations, "ruleUrl", externAlert.getRuleUrl());
|
||||
putIfNotBlank(annotations, "entityUrl", externAlert.getEntityUrl());
|
||||
putIfNotBlank(annotations, "alertRuleUrl", externAlert.getAlertRuleUrl());
|
||||
putIfNotBlank(annotations, "alertHistoryUrl", externAlert.getAlertHistoryUrl());
|
||||
return annotations;
|
||||
}
|
||||
|
||||
private String getAlertContent(AlibabaCloudCmsExternAlert externAlert) {
|
||||
if (StringUtils.isNotBlank(externAlert.getAlertMessage())) {
|
||||
return externAlert.getAlertMessage();
|
||||
}
|
||||
if (StringUtils.isNotBlank(externAlert.getSubject())) {
|
||||
return externAlert.getSubject();
|
||||
}
|
||||
return "Alibaba Cloud Monitor alert";
|
||||
}
|
||||
|
||||
private String convertSeverity(String severity) {
|
||||
if (StringUtils.isBlank(severity)) {
|
||||
return null;
|
||||
}
|
||||
return switch (severity.toUpperCase(Locale.ROOT)) {
|
||||
case "EMERGENCY" -> CommonConstants.ALERT_SEVERITY_EMERGENCY;
|
||||
case "CRITICAL" -> CommonConstants.ALERT_SEVERITY_CRITICAL;
|
||||
case "WARN", "WARNING" -> CommonConstants.ALERT_SEVERITY_WARNING;
|
||||
case "INFO", "INFORMATIONAL" -> CommonConstants.ALERT_SEVERITY_INFO;
|
||||
default -> severity.toLowerCase(Locale.ROOT);
|
||||
};
|
||||
}
|
||||
|
||||
private void putValues(Map<String, String> target, Map<String, Object> values) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
values.forEach((key, value) -> putValue(target, key, value));
|
||||
}
|
||||
|
||||
private void putValue(Map<String, String> target, String key, Object value) {
|
||||
if (StringUtils.isBlank(key) || value == null) {
|
||||
return;
|
||||
}
|
||||
String stringValue;
|
||||
if (value instanceof Map<?, ?> || value instanceof Collection<?>) {
|
||||
stringValue = JsonUtil.toJson(value);
|
||||
} else {
|
||||
stringValue = String.valueOf(value);
|
||||
}
|
||||
putIfNotBlank(target, key, stringValue);
|
||||
}
|
||||
|
||||
private void putIfNotBlank(Map<String, String> target, String key, String value) {
|
||||
if (StringUtils.isNotBlank(value)) {
|
||||
target.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
+88
@@ -27,11 +27,13 @@ import org.apache.hertzbeat.alert.service.AlertDefineService;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.common.queue.impl.InMemoryCommonDataQueue;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
|
||||
@@ -43,6 +45,7 @@ import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -254,4 +257,89 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
|
||||
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEmptyStringFieldStillTriggersAlert() throws InterruptedException {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
builder.setId(518679137103104L)
|
||||
.setApp("fullsite")
|
||||
.setMetrics("summary")
|
||||
.setPriority(1)
|
||||
.setCode(CollectRep.Code.SUCCESS);
|
||||
|
||||
CollectRep.Field url = CollectRep.Field.newBuilder().setName("url").setType(CommonConstants.TYPE_STRING).setLabel(true).build();
|
||||
CollectRep.Field statusCode = CollectRep.Field.newBuilder().setName("statusCode").setType(CommonConstants.TYPE_STRING).build();
|
||||
CollectRep.Field errorMsg = CollectRep.Field.newBuilder().setName("errorMsg").setType(CommonConstants.TYPE_STRING).build();
|
||||
|
||||
Map<String, String> meta = new HashMap<>();
|
||||
meta.put(MetricDataConstants.INSTANCE_NAME, "site");
|
||||
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
|
||||
|
||||
builder.addMetadataAll(meta);
|
||||
builder.addAllFields(Lists.newArrayList(url, statusCode, errorMsg));
|
||||
builder.addValueRow(CollectRep.ValueRow.newBuilder()
|
||||
.addColumn("https://example.com/broken").addColumn("404").addColumn("").build());
|
||||
|
||||
CollectRep.MetricsData metricsData = builder.build();
|
||||
|
||||
AlertDefine matchDefine = new AlertDefine();
|
||||
matchDefine.setId(1L);
|
||||
matchDefine.setName("sitemap-status");
|
||||
matchDefine.setExpr("equals(__app__,\"fullsite\") && !matches(statusCode,\"^2[0-9]+\") && !contains(errorMsg,\"timed out\")");
|
||||
matchDefine.setTemplate("site down: ${url}");
|
||||
matchDefine.setTimes(1);
|
||||
|
||||
when(alertDefineService.getMetricsRealTimeAlertDefines()).thenReturn(Collections.singletonList(matchDefine));
|
||||
when(dataQueue.pollMetricsDataToAlerter()).thenReturn(metricsData).thenThrow(new InterruptedException());
|
||||
|
||||
metricsRealTimeAlertCalculator.startCalculate();
|
||||
|
||||
Thread.sleep(3000);
|
||||
|
||||
ArgumentCaptor<SingleAlert> alertCaptor = ArgumentCaptor.forClass(SingleAlert.class);
|
||||
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(alertCaptor.capture());
|
||||
assertEquals("site down: https://example.com/broken", alertCaptor.getValue().getContent());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testUnparseableNumberFieldDoesNotAbortRule() throws InterruptedException {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
builder.setId(518679137103105L)
|
||||
.setApp("fullsite")
|
||||
.setMetrics("summary")
|
||||
.setPriority(1)
|
||||
.setCode(CollectRep.Code.SUCCESS);
|
||||
|
||||
CollectRep.Field url = CollectRep.Field.newBuilder().setName("url").setType(CommonConstants.TYPE_STRING).setLabel(true).build();
|
||||
CollectRep.Field responseTime = CollectRep.Field.newBuilder().setName("responseTime").setType(CommonConstants.TYPE_NUMBER).build();
|
||||
|
||||
Map<String, String> meta = new HashMap<>();
|
||||
meta.put(MetricDataConstants.INSTANCE_NAME, "site");
|
||||
meta.put(MetricDataConstants.INSTANCE, "127.0.0.1");
|
||||
|
||||
builder.addMetadataAll(meta);
|
||||
builder.addAllFields(Lists.newArrayList(url, responseTime));
|
||||
builder.addValueRow(CollectRep.ValueRow.newBuilder()
|
||||
.addColumn("https://example.com/a").addColumn("").build());
|
||||
|
||||
CollectRep.MetricsData metricsData = builder.build();
|
||||
|
||||
AlertDefine guardedDefine = new AlertDefine();
|
||||
guardedDefine.setId(2L);
|
||||
guardedDefine.setName("slow-site");
|
||||
guardedDefine.setExpr("equals(__app__,\"fullsite\") && exists(responseTime) && responseTime > 100");
|
||||
guardedDefine.setTemplate("slow: ${url}");
|
||||
guardedDefine.setTimes(1);
|
||||
|
||||
when(alertDefineService.getMetricsRealTimeAlertDefines()).thenReturn(Collections.singletonList(guardedDefine));
|
||||
when(dataQueue.pollMetricsDataToAlerter()).thenReturn(metricsData).thenThrow(new InterruptedException());
|
||||
|
||||
metricsRealTimeAlertCalculator.startCalculate();
|
||||
|
||||
Thread.sleep(3000);
|
||||
|
||||
// unparseable number is defined as null: exists() short-circuits to false, no alarm and no abort
|
||||
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any());
|
||||
verify(alarmCacheManager, times(1)).removeFiring(any(), any());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
@@ -69,6 +69,7 @@ public class AlertDefineExcelImExportServiceTest {
|
||||
row.createCell(6).setCellValue(JsonUtil.toJson(Map.of("key", "value")));
|
||||
row.createCell(7).setCellValue("template1");
|
||||
row.createCell(8).setCellValue(true);
|
||||
row.createCell(9).setCellValue("promql");
|
||||
|
||||
ByteArrayInputStream inputStream = new ByteArrayInputStream(toByteArray(initialWorkbook));
|
||||
|
||||
@@ -93,6 +94,7 @@ public class AlertDefineExcelImExportServiceTest {
|
||||
assertEquals(Map.of("key", "value"), alertDefineDTO.getAnnotations());
|
||||
assertEquals("template1", alertDefineDTO.getTemplate());
|
||||
assertTrue(alertDefineDTO.getEnable());
|
||||
assertEquals("promql", alertDefineDTO.getDatasource());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +113,7 @@ public class AlertDefineExcelImExportServiceTest {
|
||||
alertDefineDTO.setAnnotations(Map.of("key", "value"));
|
||||
alertDefineDTO.setTemplate("template1");
|
||||
alertDefineDTO.setEnable(true);
|
||||
alertDefineDTO.setDatasource("promql");
|
||||
exportAlertDefineDTO.setAlertDefine(alertDefineDTO);
|
||||
exportAlertDefineList.add(exportAlertDefineDTO);
|
||||
|
||||
@@ -129,6 +132,7 @@ public class AlertDefineExcelImExportServiceTest {
|
||||
assertEquals("Annotations", headerRow.getCell(6).getStringCellValue());
|
||||
assertEquals("Template", headerRow.getCell(7).getStringCellValue());
|
||||
assertEquals("Enable", headerRow.getCell(8).getStringCellValue());
|
||||
assertEquals("Datasource", headerRow.getCell(9).getStringCellValue());
|
||||
|
||||
Row dataRow = resultSheet.getRow(1);
|
||||
assertEquals("app1", dataRow.getCell(0).getStringCellValue());
|
||||
@@ -140,6 +144,7 @@ public class AlertDefineExcelImExportServiceTest {
|
||||
assertEquals(JsonUtil.toJson(Map.of("key", "value")), dataRow.getCell(6).getStringCellValue());
|
||||
assertEquals("template1", dataRow.getCell(7).getStringCellValue());
|
||||
assertTrue(dataRow.getCell(8).getBooleanCellValue());
|
||||
assertEquals("promql", dataRow.getCell(9).getStringCellValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+29
-1
@@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
@@ -29,8 +31,10 @@ import java.util.List;
|
||||
import org.apache.hertzbeat.alert.dto.AlertDefineDTO;
|
||||
import org.apache.hertzbeat.alert.dto.ExportAlertDefineDTO;
|
||||
import org.apache.hertzbeat.alert.service.impl.AlertDefineJsonImExportServiceImpl;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
/**
|
||||
* test case for {@link AlertDefineJsonImExportServiceImpl}
|
||||
@@ -43,7 +47,7 @@ class AlertDefineJsonImExportServiceTest {
|
||||
@SuppressWarnings("checkstyle:OperatorWrap")
|
||||
private static final String JSON_DATA = "[{\"alertDefine\":{\"name\":\"App1\",\"type\":\"realtime\"," +
|
||||
"\"expr\":\"Expr1\",\"period\":3000,\"times\":3," +
|
||||
"\"enable\":true,\"template\":\"Template1\"}}]";
|
||||
"\"enable\":true,\"template\":\"Template1\",\"datasource\":\"promql\"}}]";
|
||||
|
||||
private InputStream inputStream;
|
||||
private List<ExportAlertDefineDTO> alertDefineList;
|
||||
@@ -77,6 +81,7 @@ class AlertDefineJsonImExportServiceTest {
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("App1", result.get(0).getAlertDefine().getName());
|
||||
assertEquals("realtime", result.get(0).getAlertDefine().getType());
|
||||
assertEquals("promql", result.get(0).getAlertDefine().getDatasource());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -100,6 +105,29 @@ class AlertDefineJsonImExportServiceTest {
|
||||
assertTrue(result.contains("realtime"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExportKeepsDatasource() {
|
||||
AlertDefineService alertDefineService = mock(AlertDefineService.class);
|
||||
AlertDefine define = AlertDefine.builder()
|
||||
.name("test")
|
||||
.type("periodic_metric")
|
||||
.expr("cpu_usage{instance=\"server1\"} > 80")
|
||||
.datasource("promql")
|
||||
.period(300)
|
||||
.times(3)
|
||||
.template("test")
|
||||
.enable(true)
|
||||
.build();
|
||||
when(alertDefineService.getAlertDefine(1L)).thenReturn(define);
|
||||
ReflectionTestUtils.setField(service, "alertDefineService", alertDefineService);
|
||||
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
service.exportConfig(outputStream, List.of(1L));
|
||||
|
||||
String result = outputStream.toString(StandardCharsets.UTF_8);
|
||||
assertTrue(result.contains("promql"), "exported config should keep datasource, but got: " + result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testType() {
|
||||
assertEquals("JSON", service.type());
|
||||
|
||||
+3
@@ -64,6 +64,7 @@ class AlertDefineYamlImExportServiceTest {
|
||||
times: 3
|
||||
enable: true
|
||||
template: Template1
|
||||
datasource: promql
|
||||
""";
|
||||
|
||||
private InputStream inputStream;
|
||||
@@ -82,6 +83,7 @@ class AlertDefineYamlImExportServiceTest {
|
||||
alertDefine.setExpr("Expr1");
|
||||
alertDefine.setEnable(true);
|
||||
alertDefine.setTemplate("Template1");
|
||||
alertDefine.setDatasource("promql");
|
||||
|
||||
ExportAlertDefineDTO exportAlertDefine = new ExportAlertDefineDTO();
|
||||
exportAlertDefine.setAlertDefine(alertDefine);
|
||||
@@ -135,6 +137,7 @@ class AlertDefineYamlImExportServiceTest {
|
||||
assertTrue(yamlOutput.contains("name: App1"));
|
||||
assertTrue(yamlOutput.contains("type: realtime"));
|
||||
assertTrue(yamlOutput.contains("expr: Expr1"));
|
||||
assertTrue(yamlOutput.contains("datasource: promql"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.alert.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import java.time.Instant;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
|
||||
import org.apache.hertzbeat.alert.service.impl.AlibabaCloudCmsExternAlertService;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
/**
|
||||
* Unit test for {@link AlibabaCloudCmsExternAlertService}.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class AlibabaCloudCmsExternAlertServiceTest {
|
||||
|
||||
private static final long EVENT_TIME = 1785300000123L;
|
||||
|
||||
@Mock
|
||||
private AlarmCommonReduce alarmCommonReduce;
|
||||
|
||||
private AlibabaCloudCmsExternAlertService externAlertService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
externAlertService = new AlibabaCloudCmsExternAlertService(alarmCommonReduce);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldConvertTriggeredAlert() {
|
||||
externAlertService.addExternAlert("""
|
||||
{
|
||||
"specversion": "1.0",
|
||||
"id": "alert-event-1",
|
||||
"type": "ALERT",
|
||||
"subtype": "NORMAL_TRIGGER",
|
||||
"time": "2026-07-29T06:00:00Z",
|
||||
"timestamp": 1785300000123,
|
||||
"subject": "ECS CPU usage is high",
|
||||
"severity": "WARNING",
|
||||
"status": "OCCURRED",
|
||||
"userId": "123456",
|
||||
"ruleId": "rule-1",
|
||||
"workspace": "default-cms-123456-cn-hangzhou",
|
||||
"traceId": "trace-1",
|
||||
"alertMessage": "CPU usage exceeded 80%",
|
||||
"alertEntityId": "ecs:i-123",
|
||||
"resource": {
|
||||
"entity": {
|
||||
"domain": "ecs",
|
||||
"entity_type": "instance",
|
||||
"entity_id": "i-123",
|
||||
"prop": {
|
||||
"instanceName": "api-server"
|
||||
}
|
||||
},
|
||||
"tags": {
|
||||
"regionId": "cn-hangzhou",
|
||||
"environment": "production"
|
||||
}
|
||||
},
|
||||
"labels": {
|
||||
"_cms_region": "cn-hangzhou",
|
||||
"customNumber": 7
|
||||
},
|
||||
"annotations": {
|
||||
"current_value": "92.5"
|
||||
},
|
||||
"data": {
|
||||
"value": 92.5,
|
||||
"threshold": 80,
|
||||
"comparisonOperator": ">"
|
||||
},
|
||||
"alertEntityFields": {
|
||||
"privateIp": "10.0.0.1"
|
||||
},
|
||||
"alertHistoryUrl": "https://cmsnext.console.aliyun.com/history",
|
||||
"futureField": "ignored"
|
||||
}
|
||||
""");
|
||||
|
||||
SingleAlert alert = captureAlert();
|
||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, alert.getStatus());
|
||||
assertEquals(EVENT_TIME, alert.getStartAt());
|
||||
assertEquals(EVENT_TIME, alert.getActiveAt());
|
||||
assertNull(alert.getEndAt());
|
||||
assertEquals("CPU usage exceeded 80%", alert.getContent());
|
||||
assertEquals("alibabacloud-cms", alert.getLabels().get("__source__"));
|
||||
assertEquals("ECS CPU usage is high", alert.getLabels().get("alertname"));
|
||||
assertEquals(CommonConstants.ALERT_SEVERITY_WARNING, alert.getLabels().get("severity"));
|
||||
assertEquals("instance", alert.getLabels().get("resourceType"));
|
||||
assertEquals("i-123", alert.getLabels().get("resourceId"));
|
||||
assertEquals("7", alert.getLabels().get("customNumber"));
|
||||
assertEquals("api-server", alert.getAnnotations().get("instanceName"));
|
||||
assertEquals("92.5", alert.getAnnotations().get("value"));
|
||||
assertEquals("80", alert.getAnnotations().get("threshold"));
|
||||
assertEquals("10.0.0.1", alert.getAnnotations().get("privateIp"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldConvertResolvedAlertAndIsoTime() {
|
||||
externAlertService.addExternAlert("""
|
||||
{
|
||||
"subtype": "NORMAL_RESOLVE",
|
||||
"time": "2026-07-29T06:00:00Z",
|
||||
"subject": "ECS CPU usage is high",
|
||||
"severity": "CRITICAL",
|
||||
"status": "RESOLVED",
|
||||
"labels": {
|
||||
"instanceId": "i-123"
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
SingleAlert alert = captureAlert();
|
||||
long expectedTime = Instant.parse("2026-07-29T06:00:00Z").toEpochMilli();
|
||||
assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alert.getStatus());
|
||||
assertEquals(expectedTime, alert.getStartAt());
|
||||
assertNull(alert.getActiveAt());
|
||||
assertEquals(expectedTime, alert.getEndAt());
|
||||
assertEquals("ECS CPU usage is high", alert.getContent());
|
||||
assertEquals(CommonConstants.ALERT_SEVERITY_CRITICAL, alert.getLabels().get("severity"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldTreatRecoveredStatusAsResolved() {
|
||||
externAlertService.addExternAlert("""
|
||||
{
|
||||
"timestamp": 1785300000123,
|
||||
"subject": "Recovered alert",
|
||||
"status": "RECOVERED"
|
||||
}
|
||||
""");
|
||||
|
||||
SingleAlert alert = captureAlert();
|
||||
assertEquals(CommonConstants.ALERT_STATUS_RESOLVED, alert.getStatus());
|
||||
assertEquals(EVENT_TIME, alert.getEndAt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldIgnoreInvalidPayload() {
|
||||
externAlertService.addExternAlert("invalid json");
|
||||
externAlertService.addExternAlert("{\"subject\":\"missing status\"}");
|
||||
|
||||
verify(alarmCommonReduce, never()).reduceAndSendAlarm(any(SingleAlert.class));
|
||||
assertEquals("alibabacloud-cms", externAlertService.supportSource());
|
||||
}
|
||||
|
||||
private SingleAlert captureAlert() {
|
||||
ArgumentCaptor<SingleAlert> captor = ArgumentCaptor.forClass(SingleAlert.class);
|
||||
verify(alarmCommonReduce).reduceAndSendAlarm(captor.capture());
|
||||
return captor.getValue();
|
||||
}
|
||||
}
|
||||
@@ -151,8 +151,8 @@
|
||||
<!--Bouncy Castle-->
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcpkix-jdk15on</artifactId>
|
||||
<version>1.68</version>
|
||||
<artifactId>bcpkix-jdk18on</artifactId>
|
||||
<version>${bouncycastle.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.common;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.collector.constants.CollectorConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* Shared one-row response handling for command-based collectors.
|
||||
*/
|
||||
public final class OneRowResponseSupport {
|
||||
|
||||
/**
|
||||
* Parse type where each output line maps to one alias field of a single result row.
|
||||
*/
|
||||
public static final String PARSE_TYPE_ONE_ROW = "oneRow";
|
||||
|
||||
private OneRowResponseSupport() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Treat blank stdout without an error signal (no stderr, exit status present and <= 1,
|
||||
* grep-style no match) as valid empty one-row data: append a row of null placeholders so the
|
||||
* metric stays visible and alertable.
|
||||
*
|
||||
* @return true if handled as empty success, false if the caller should report a failure
|
||||
*/
|
||||
public static boolean tryAppendEmptyOneRow(String parseType, String stdErr, Integer exitStatus,
|
||||
List<String> aliasFields, CollectRep.MetricsData.Builder builder,
|
||||
Long responseTime) {
|
||||
if (PARSE_TYPE_ONE_ROW.equals(parseType)
|
||||
&& !StringUtils.hasText(stdErr)
|
||||
&& exitStatus != null && exitStatus <= 1) {
|
||||
appendEmptyValues(aliasFields, builder, responseTime);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the failure message for a command that produced no usable stdout: prefer the captured
|
||||
* stderr, then a non-trivial exit status, otherwise the generic null-data message.
|
||||
*/
|
||||
public static String buildBlankFailureMessage(String stdErr, Integer exitStatus,
|
||||
String exitCodePrefix, String nullMessage) {
|
||||
if (StringUtils.hasText(stdErr)) {
|
||||
return stdErr.trim();
|
||||
}
|
||||
if (exitStatus != null && exitStatus > 1) {
|
||||
return exitCodePrefix + exitStatus;
|
||||
}
|
||||
return nullMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map each output line to one alias field of a single row; missing trailing lines become
|
||||
* NULL_VALUE columns so a partial result keeps its values and the gap stays alertable.
|
||||
*/
|
||||
public static void appendResponseValues(String result, List<String> aliasFields,
|
||||
CollectRep.MetricsData.Builder builder, Long responseTime) {
|
||||
List<String> safeAliasFields = aliasFields == null ? Collections.emptyList() : aliasFields;
|
||||
String[] lines = result.split("\n");
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
int aliasIndex = 0;
|
||||
int lineIndex = 0;
|
||||
while (aliasIndex < safeAliasFields.size()) {
|
||||
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(safeAliasFields.get(aliasIndex))) {
|
||||
valueRowBuilder.addColumn(responseTime.toString());
|
||||
} else {
|
||||
if (lineIndex < lines.length) {
|
||||
valueRowBuilder.addColumn(lines[lineIndex].trim());
|
||||
} else {
|
||||
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
|
||||
}
|
||||
lineIndex++;
|
||||
}
|
||||
aliasIndex++;
|
||||
}
|
||||
builder.addValueRow(valueRowBuilder.build());
|
||||
}
|
||||
|
||||
public static void appendEmptyValues(List<String> aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) {
|
||||
List<String> safeAliasFields = aliasFields == null ? Collections.emptyList() : aliasFields;
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
for (String aliasField : safeAliasFields) {
|
||||
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasField)) {
|
||||
valueRowBuilder.addColumn(responseTime.toString());
|
||||
} else {
|
||||
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
|
||||
}
|
||||
}
|
||||
builder.addValueRow(valueRowBuilder.build());
|
||||
}
|
||||
}
|
||||
+5
-7
@@ -704,13 +704,11 @@ public class HttpCollectImpl extends AbstractCollect {
|
||||
valueRowBuilder.addColumn(String.valueOf(value));
|
||||
} else {
|
||||
if (alias.startsWith("$.")) {
|
||||
List<Object> subResults = JsonPathParser.parseContentWithJsonPath(resp, http.getParseScript() + alias.substring(1));
|
||||
if (subResults != null && subResults.size() > i) {
|
||||
Object resultValue = subResults.get(i);
|
||||
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
|
||||
} else {
|
||||
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
|
||||
}
|
||||
// per-row evaluation, a global "parseScript + alias" query would misalign rows missing the path
|
||||
List<Object> aliasValues = JsonPathParser.parseRowWithJsonPath(objectValue, alias);
|
||||
// a wildcard alias matching multiple values is kept whole and rendered as "[v1, v2]"
|
||||
Object resultValue = aliasValues.size() == 1 ? aliasValues.get(0) : (aliasValues.isEmpty() ? null : aliasValues);
|
||||
valueRowBuilder.addColumn(resultValue == null ? CommonConstants.NULL_VALUE : String.valueOf(resultValue));
|
||||
} else {
|
||||
addColumnForSummary(responseTime, valueRowBuilder, keywordNum, alias);
|
||||
}
|
||||
|
||||
+33
-2
@@ -60,16 +60,37 @@ public class OnlineParser {
|
||||
}
|
||||
|
||||
public static Map<String, MetricFamily> parseMetrics(InputStream inputStream) throws IOException {
|
||||
Map<String, MetricFamily> metricFamilyMap = new ConcurrentHashMap<>(10);
|
||||
return parseMetrics(inputStream, Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses at most {@code maxSamples} samples from the supplied stream.
|
||||
*
|
||||
* @param inputStream The Prometheus text stream
|
||||
* @param maxSamples The maximum number of samples to materialize
|
||||
* @return The parsed metric families, or {@code null} when the text format is invalid
|
||||
* @throws IOException When the stream cannot be read
|
||||
* @throws SampleLimitExceededException When another sample follows the configured limit
|
||||
*/
|
||||
public static Map<String, MetricFamily> parseMetrics(InputStream inputStream, int maxSamples) throws IOException {
|
||||
if (maxSamples < 0) {
|
||||
throw new IllegalArgumentException("maxSamples must not be negative");
|
||||
}
|
||||
final Map<String, MetricFamily> metricFamilyMap = new ConcurrentHashMap<>(10);
|
||||
int sampleCount = 0;
|
||||
try {
|
||||
int i = getChar(inputStream);
|
||||
while (i != -1) {
|
||||
if (i == '#' || i == '\n') {
|
||||
skipToLineEnd(inputStream).maybeEol().maybeEof().noElse();
|
||||
} else {
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
if (sampleCount >= maxSamples) {
|
||||
throw new SampleLimitExceededException(maxSamples);
|
||||
}
|
||||
final StringBuilder stringBuilder = new StringBuilder();
|
||||
stringBuilder.append((char) i);
|
||||
parseMetric(inputStream, metricFamilyMap, stringBuilder);
|
||||
sampleCount++;
|
||||
}
|
||||
i = getChar(inputStream);
|
||||
// To address the `\n\r` scenario, it is necessary to skip
|
||||
@@ -84,6 +105,16 @@ public class OnlineParser {
|
||||
return metricFamilyMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Signals that parsing stopped before materializing a sample beyond the configured limit.
|
||||
*/
|
||||
public static final class SampleLimitExceededException extends IOException {
|
||||
|
||||
public SampleLimitExceededException(int limit) {
|
||||
super("prometheus payload exceeds the " + limit + " sample limit");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses Prometheus metrics from the given {@link InputStream}, but only for the specified metric name.
|
||||
* <p>
|
||||
|
||||
+46
-33
@@ -30,6 +30,7 @@ import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollect;
|
||||
import org.apache.hertzbeat.collector.collect.common.OneRowResponseSupport;
|
||||
import org.apache.hertzbeat.collector.constants.CollectorConstants;
|
||||
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
@@ -52,7 +53,6 @@ public class ScriptCollectImpl extends AbstractCollect {
|
||||
private static final String BASH_C = "-c";
|
||||
private static final String POWERSHELL_C = "-Command";
|
||||
private static final String POWERSHELL_FILE = "-File";
|
||||
private static final String PARSE_TYPE_ONE_ROW = "oneRow";
|
||||
private static final String PARSE_TYPE_MULTI_ROW = "multiRow";
|
||||
private static final String PARSE_TYPE_NETCAT = "netcat";
|
||||
private static final String PARSE_TYPE_LOG = "log";
|
||||
@@ -113,25 +113,48 @@ public class ScriptCollectImpl extends AbstractCollect {
|
||||
try {
|
||||
Process process = processBuilder.start();
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.forName(scriptProtocol.getCharset())));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (StringUtils.hasText(line)) {
|
||||
response.append(line).append("\n");
|
||||
BufferedReader errorReader = new BufferedReader(
|
||||
new InputStreamReader(process.getErrorStream(), Charset.forName(scriptProtocol.getCharset())));
|
||||
// drain stderr on its own thread: a full stderr pipe would deadlock the stdout read;
|
||||
// StringBuffer because the drainer may still be writing when the buffer is read
|
||||
StringBuffer errorBuffer = new StringBuffer();
|
||||
Thread errorDrainer = new Thread(() -> {
|
||||
try {
|
||||
String errorLine;
|
||||
while ((errorLine = errorReader.readLine()) != null) {
|
||||
if (StringUtils.hasText(errorLine)) {
|
||||
errorBuffer.append(errorLine).append("\n");
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("read script error stream failed: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
process.waitFor();
|
||||
});
|
||||
errorDrainer.setDaemon(true);
|
||||
errorDrainer.start();
|
||||
String result = readResponse(reader);
|
||||
int exitCode = process.waitFor();
|
||||
// bounded: a lingering grandchild can keep the stderr pipe open
|
||||
errorDrainer.join(1000);
|
||||
Long responseTime = System.currentTimeMillis() - startTime;
|
||||
String result = String.valueOf(response);
|
||||
String errorResult = errorBuffer.toString();
|
||||
if (!StringUtils.hasText(result)) {
|
||||
if (OneRowResponseSupport.tryAppendEmptyOneRow(scriptProtocol.getParseType(), errorResult,
|
||||
exitCode, metrics.getAliasFields(), builder, responseTime)) {
|
||||
return;
|
||||
}
|
||||
builder.setCode(CollectRep.Code.FAIL);
|
||||
builder.setMsg("Script response data is null");
|
||||
builder.setMsg(OneRowResponseSupport.buildBlankFailureMessage(errorResult, exitCode,
|
||||
"Script exited with code: ", "Script response data is null"));
|
||||
return;
|
||||
}
|
||||
if (StringUtils.hasText(errorResult)) {
|
||||
log.warn("script command succeeded but wrote to stderr: {}", errorResult.trim());
|
||||
}
|
||||
switch (scriptProtocol.getParseType()) {
|
||||
case PARSE_TYPE_LOG -> parseResponseDataByLog(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case PARSE_TYPE_NETCAT -> parseResponseDataByNetcat(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case OneRowResponseSupport.PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case PARSE_TYPE_MULTI_ROW -> parseResponseDataByMulti(result, metrics.getAliasFields(), builder, responseTime);
|
||||
default -> {
|
||||
builder.setCode(CollectRep.Code.FAIL);
|
||||
@@ -207,28 +230,7 @@ public class ScriptCollectImpl extends AbstractCollect {
|
||||
}
|
||||
|
||||
private void parseResponseDataByOne(String result, List<String> aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) {
|
||||
String[] lines = result.split("\n");
|
||||
if (lines.length + 1 < aliasFields.size()) {
|
||||
log.error("ssh response data not enough: {}", result);
|
||||
return;
|
||||
}
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
int aliasIndex = 0;
|
||||
int lineIndex = 0;
|
||||
while (aliasIndex < aliasFields.size()) {
|
||||
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasFields.get(aliasIndex))) {
|
||||
valueRowBuilder.addColumn(responseTime.toString());
|
||||
} else {
|
||||
if (lineIndex < lines.length) {
|
||||
valueRowBuilder.addColumn(lines[lineIndex].trim());
|
||||
} else {
|
||||
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
|
||||
}
|
||||
lineIndex++;
|
||||
}
|
||||
aliasIndex++;
|
||||
}
|
||||
builder.addValueRow(valueRowBuilder.build());
|
||||
OneRowResponseSupport.appendResponseValues(result, aliasFields, builder, responseTime);
|
||||
}
|
||||
|
||||
private void parseResponseDataByMulti(String result, List<String> aliasFields,
|
||||
@@ -261,4 +263,15 @@ public class ScriptCollectImpl extends AbstractCollect {
|
||||
builder.addValueRow(valueRowBuilder.build());
|
||||
}
|
||||
}
|
||||
|
||||
private String readResponse(BufferedReader reader) throws IOException {
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
if (StringUtils.hasText(line)) {
|
||||
response.append(line).append("\n");
|
||||
}
|
||||
}
|
||||
return response.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+21
-28
@@ -21,6 +21,8 @@ import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.net.ConnectException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.security.GeneralSecurityException;
|
||||
import java.util.ArrayList;
|
||||
@@ -33,6 +35,7 @@ import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollect;
|
||||
import org.apache.hertzbeat.collector.collect.common.OneRowResponseSupport;
|
||||
import org.apache.hertzbeat.collector.collect.common.ssh.CommonSshBlacklist;
|
||||
import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper;
|
||||
import org.apache.hertzbeat.collector.constants.CollectorConstants;
|
||||
@@ -49,7 +52,6 @@ import org.apache.sshd.client.session.ClientSession;
|
||||
import org.apache.sshd.common.SshException;
|
||||
import org.apache.sshd.common.channel.exception.SshChannelOpenException;
|
||||
import org.apache.sshd.common.future.CloseFuture;
|
||||
import org.apache.sshd.common.util.io.output.NoCloseOutputStream;
|
||||
import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
@@ -58,7 +60,6 @@ import org.springframework.util.StringUtils;
|
||||
@Slf4j
|
||||
public class SshCollectImpl extends AbstractCollect {
|
||||
|
||||
private static final String PARSE_TYPE_ONE_ROW = "oneRow";
|
||||
private static final String PARSE_TYPE_MULTI_ROW = "multiRow";
|
||||
private static final String PARSE_TYPE_NETCAT = "netcat";
|
||||
private static final String PARSE_TYPE_LOG = "log";
|
||||
@@ -93,8 +94,9 @@ public class SshCollectImpl extends AbstractCollect {
|
||||
}
|
||||
channel = clientSession.createExecChannel(sshProtocol.getScript());
|
||||
ByteArrayOutputStream response = new ByteArrayOutputStream();
|
||||
ByteArrayOutputStream errorResponse = new ByteArrayOutputStream();
|
||||
channel.setOut(response);
|
||||
channel.setErr(new NoCloseOutputStream(System.err));
|
||||
channel.setErr(errorResponse);
|
||||
channel.open().verify(timeout);
|
||||
List<ClientChannelEvent> list = new ArrayList<>();
|
||||
list.add(ClientChannelEvent.CLOSED);
|
||||
@@ -107,16 +109,28 @@ public class SshCollectImpl extends AbstractCollect {
|
||||
throw new SocketTimeoutException("Failed to retrieve command result in time: " + sshProtocol.getScript());
|
||||
}
|
||||
Long responseTime = System.currentTimeMillis() - startTime;
|
||||
String result = response.toString();
|
||||
Charset charset = StringUtils.hasText(sshProtocol.getCharset())
|
||||
? Charset.forName(sshProtocol.getCharset()) : StandardCharsets.UTF_8;
|
||||
String result = response.toString(charset);
|
||||
String errorResult = errorResponse.toString(charset);
|
||||
Integer exitStatus = channel.getExitStatus();
|
||||
if (!StringUtils.hasText(result)) {
|
||||
if (OneRowResponseSupport.tryAppendEmptyOneRow(sshProtocol.getParseType(), errorResult,
|
||||
exitStatus, metrics.getAliasFields(), builder, responseTime)) {
|
||||
return;
|
||||
}
|
||||
builder.setCode(CollectRep.Code.FAIL);
|
||||
builder.setMsg("ssh shell response data is null");
|
||||
builder.setMsg(OneRowResponseSupport.buildBlankFailureMessage(errorResult, exitStatus,
|
||||
"ssh command exited with code: ", "ssh shell response data is null"));
|
||||
return;
|
||||
}
|
||||
if (StringUtils.hasText(errorResult)) {
|
||||
log.warn("ssh command succeeded but wrote to stderr: {}", errorResult.trim());
|
||||
}
|
||||
switch (sshProtocol.getParseType()) {
|
||||
case PARSE_TYPE_LOG -> parseResponseDataByLog(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case PARSE_TYPE_NETCAT -> parseResponseDataByNetcat(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case OneRowResponseSupport.PARSE_TYPE_ONE_ROW -> parseResponseDataByOne(result, metrics.getAliasFields(), builder, responseTime);
|
||||
case PARSE_TYPE_MULTI_ROW -> parseResponseDataByMulti(result, metrics.getAliasFields(), builder, responseTime);
|
||||
default -> {
|
||||
builder.setCode(CollectRep.Code.FAIL);
|
||||
@@ -244,28 +258,7 @@ public class SshCollectImpl extends AbstractCollect {
|
||||
}
|
||||
|
||||
private void parseResponseDataByOne(String result, List<String> aliasFields, CollectRep.MetricsData.Builder builder, Long responseTime) {
|
||||
String[] lines = result.split("\n");
|
||||
if (lines.length + 1 < aliasFields.size()) {
|
||||
log.error("ssh response data not enough: {}", result);
|
||||
return;
|
||||
}
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
int aliasIndex = 0;
|
||||
int lineIndex = 0;
|
||||
while (aliasIndex < aliasFields.size()) {
|
||||
if (CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(aliasFields.get(aliasIndex))) {
|
||||
valueRowBuilder.addColumn(responseTime.toString());
|
||||
} else {
|
||||
if (lineIndex < lines.length) {
|
||||
valueRowBuilder.addColumn(lines[lineIndex].trim());
|
||||
} else {
|
||||
valueRowBuilder.addColumn(CommonConstants.NULL_VALUE);
|
||||
}
|
||||
lineIndex++;
|
||||
}
|
||||
aliasIndex++;
|
||||
}
|
||||
builder.addValueRow(valueRowBuilder.build());
|
||||
OneRowResponseSupport.appendResponseValues(result, aliasFields, builder, responseTime);
|
||||
}
|
||||
|
||||
private void parseResponseDataByMulti(String result, List<String> aliasFields,
|
||||
|
||||
+30
-18
@@ -67,14 +67,20 @@ public class TelnetCollectImpl extends AbstractCollect {
|
||||
long responseTime = System.currentTimeMillis() - startTime;
|
||||
List<String> aliasFields = metrics.getAliasFields();
|
||||
String app = builder.getApp();
|
||||
Map<String, String> resultMap = execCmdAndParseResult(telnetClient, telnet.getCmd(), app);
|
||||
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
|
||||
if (resultMap.size() < aliasFields.size()) {
|
||||
log.error("telnet response data not enough: {}", resultMap);
|
||||
CmdResult cmdResult = execCmdAndParseResult(telnetClient, telnet.getCmd(), app);
|
||||
Map<String, String> resultMap = cmdResult.values();
|
||||
boolean expectsCmdMetrics = StringUtils.isNotBlank(telnet.getCmd())
|
||||
&& aliasFields.stream().anyMatch(field -> !CollectorConstants.RESPONSE_TIME.equalsIgnoreCase(field));
|
||||
boolean hasExpectedMetric = aliasFields.stream().anyMatch(resultMap::containsKey);
|
||||
if (expectsCmdMetrics && !hasExpectedMetric) {
|
||||
// e.g. zookeeper refusing a 4lw command not in its 4lw.commands.whitelist
|
||||
String reply = sanitizeReply(cmdResult.rawResponse());
|
||||
log.warn("telnet cmd [{}] returned no expected metrics: {}", telnet.getCmd(), reply);
|
||||
builder.setCode(CollectRep.Code.FAIL);
|
||||
builder.setMsg("The cmd execution results do not match the expected number of metrics.");
|
||||
builder.setMsg("Cmd [" + telnet.getCmd() + "] returned no expected metrics. Response: " + reply);
|
||||
return;
|
||||
}
|
||||
resultMap.put(CollectorConstants.RESPONSE_TIME, Long.toString(responseTime));
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
for (String field : aliasFields) {
|
||||
String fieldValue = resultMap.get(field);
|
||||
@@ -118,30 +124,36 @@ public class TelnetCollectImpl extends AbstractCollect {
|
||||
return DispatchConstants.PROTOCOL_TELNET;
|
||||
}
|
||||
|
||||
private static Map<String, String> execCmdAndParseResult(TelnetClient telnetClient, String cmd, String app) throws IOException {
|
||||
record CmdResult(Map<String, String> values, String rawResponse) {
|
||||
}
|
||||
|
||||
private static String sanitizeReply(String raw) {
|
||||
return StringUtils.abbreviate(raw.trim().replaceAll("[\\p{Cntrl}]+", " "), 300);
|
||||
}
|
||||
|
||||
private static CmdResult execCmdAndParseResult(TelnetClient telnetClient, String cmd, String app) throws IOException {
|
||||
if (cmd == null || StringUtils.isEmpty(cmd.trim())) {
|
||||
return new HashMap<>(16);
|
||||
return new CmdResult(new HashMap<>(16), "");
|
||||
}
|
||||
OutputStream outputStream = telnetClient.getOutputStream();
|
||||
outputStream.write(cmd.getBytes(StandardCharsets.UTF_8));
|
||||
outputStream.flush();
|
||||
String result = new String(telnetClient.getInputStream().readAllBytes());
|
||||
String[] lines = result.split("\n");
|
||||
if (CollectorConstants.ZOOKEEPER_APP.equals(app) && CollectorConstants.ZOOKEEPER_ENVI_HEAD.equals(lines[0])) {
|
||||
if (lines.length > 0 && CollectorConstants.ZOOKEEPER_APP.equals(app)
|
||||
&& CollectorConstants.ZOOKEEPER_ENVI_HEAD.equals(lines[0])) {
|
||||
lines = Arrays.stream(lines)
|
||||
.skip(1)
|
||||
.toArray(String[]::new);
|
||||
}
|
||||
boolean contains = lines[0].contains("=");
|
||||
return Arrays.stream(lines)
|
||||
.map(item -> {
|
||||
if (contains) {
|
||||
return item.split("=");
|
||||
} else {
|
||||
return item.split("\t");
|
||||
}
|
||||
})
|
||||
if (lines.length == 0) {
|
||||
return new CmdResult(new HashMap<>(16), result);
|
||||
}
|
||||
String separator = lines[0].contains("=") ? "=" : "\t";
|
||||
Map<String, String> values = Arrays.stream(lines)
|
||||
.map(item -> item.split(separator, 2))
|
||||
.filter(item -> item.length == 2)
|
||||
.collect(Collectors.toMap(x -> x[0], x -> x[1]));
|
||||
.collect(Collectors.toMap(x -> x[0], x -> x[1], (first, second) -> first, HashMap::new));
|
||||
return new CmdResult(values, result);
|
||||
}
|
||||
}
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.common;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.collector.constants.CollectorConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class OneRowResponseSupportTest {
|
||||
|
||||
@Test
|
||||
void appendResponseValuesShouldMapColumnsInOrder() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
|
||||
OneRowResponseSupport.appendResponseValues(
|
||||
"pod-a\n5\n", List.of("pod", "restart", CollectorConstants.RESPONSE_TIME), builder, 18L);
|
||||
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("pod-a", builder.getValues(0).getColumns(0));
|
||||
assertEquals("5", builder.getValues(0).getColumns(1));
|
||||
assertEquals("18", builder.getValues(0).getColumns(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void appendResponseValuesShouldPadMissingTrailingLines() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
|
||||
OneRowResponseSupport.appendResponseValues(
|
||||
"52\n35.8033\n5%",
|
||||
List.of("cpu", "memory", "disk", "nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 18L);
|
||||
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("52", builder.getValues(0).getColumns(0));
|
||||
assertEquals("35.8033", builder.getValues(0).getColumns(1));
|
||||
assertEquals("5%", builder.getValues(0).getColumns(2));
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
|
||||
assertEquals("18", builder.getValues(0).getColumns(4));
|
||||
}
|
||||
|
||||
@Test
|
||||
void appendEmptyValuesShouldFillNullPlaceholders() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
|
||||
OneRowResponseSupport.appendEmptyValues(
|
||||
List.of("nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 12L);
|
||||
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
|
||||
assertEquals("12", builder.getValues(0).getColumns(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void tryAppendEmptyOneRowShouldAcceptGrepNoMatchExitOne() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
|
||||
// grep with no match exits 1 and writes nothing: treat as valid empty data, not a failure
|
||||
boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow(
|
||||
OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "", 1,
|
||||
List.of("nfs_mount", CollectorConstants.RESPONSE_TIME), builder, 9L);
|
||||
|
||||
assertTrue(handled);
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void tryAppendEmptyOneRowShouldRejectNullExitStatus() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
|
||||
// an absent exit status (e.g. dropped ssh channel) must be treated as a failure
|
||||
boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow(
|
||||
OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "", null,
|
||||
List.of("nfs_mount"), builder, 9L);
|
||||
|
||||
assertFalse(handled);
|
||||
assertEquals(0, builder.getValuesCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void tryAppendEmptyOneRowShouldRejectNonEmptyStderr() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
|
||||
boolean handled = OneRowResponseSupport.tryAppendEmptyOneRow(
|
||||
OneRowResponseSupport.PARSE_TYPE_ONE_ROW, "permission denied", 1,
|
||||
List.of("nfs_mount"), builder, 9L);
|
||||
|
||||
assertFalse(handled);
|
||||
assertEquals(0, builder.getValuesCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildBlankFailureMessageShouldPreferStderrThenExitCode() {
|
||||
assertEquals("permission denied", OneRowResponseSupport.buildBlankFailureMessage(
|
||||
"permission denied\n", 2, "cmd exited with code: ", "null data"));
|
||||
assertEquals("cmd exited with code: 2", OneRowResponseSupport.buildBlankFailureMessage(
|
||||
"", 2, "cmd exited with code: ", "null data"));
|
||||
assertEquals("null data", OneRowResponseSupport.buildBlankFailureMessage(
|
||||
"", 1, "cmd exited with code: ", "null data"));
|
||||
assertEquals("null data", OneRowResponseSupport.buildBlankFailureMessage(
|
||||
"", null, "cmd exited with code: ", "null data"));
|
||||
}
|
||||
}
|
||||
+43
@@ -20,6 +20,7 @@ package org.apache.hertzbeat.collector.collect.http;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
@@ -383,6 +384,48 @@ class HttpCollectImplTest {
|
||||
assertEquals("0.268751364291017", firstRow.getColumns(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseResponseByJsonPathKeepsRowAlignmentWhenAliasPathMissing() throws Exception {
|
||||
String jsonResponse = "{\"items\": ["
|
||||
+ "{\"metadata\": {\"name\": \"pod-a\"}, \"status\": {\"phase\": \"Running\","
|
||||
+ " \"containerStatuses\": [{\"name\": \"c1\", \"ready\": true, \"restartCount\": 5}]}},"
|
||||
+ "{\"metadata\": {\"name\": \"pod-b-pending\"}, \"status\": {\"phase\": \"Pending\"}},"
|
||||
+ "{\"metadata\": {\"name\": \"pod-c\"}, \"status\": {\"phase\": \"Running\","
|
||||
+ " \"containerStatuses\": [{\"name\": \"c3\", \"ready\": true, \"restartCount\": 2}]}}"
|
||||
+ "]}";
|
||||
HttpProtocol http = HttpProtocol.builder()
|
||||
.parseType(DispatchConstants.PARSE_JSON_PATH)
|
||||
.parseScript("$.items.*")
|
||||
.build();
|
||||
List<CollectRep.ValueRow> capturedRows = new ArrayList<>();
|
||||
CollectRep.MetricsData.Builder builder = new CollectRep.MetricsData.Builder() {
|
||||
@Override
|
||||
public CollectRep.MetricsData.Builder addValueRow(CollectRep.ValueRow valueRow) {
|
||||
capturedRows.add(valueRow);
|
||||
return super.addValueRow(valueRow);
|
||||
}
|
||||
};
|
||||
Method parseMethod = HttpCollectImpl.class.getDeclaredMethod(
|
||||
"parseResponseByJsonPath",
|
||||
String.class,
|
||||
List.class,
|
||||
HttpProtocol.class,
|
||||
CollectRep.MetricsData.Builder.class,
|
||||
Long.class);
|
||||
parseMethod.setAccessible(true);
|
||||
|
||||
parseMethod.invoke(httpCollectImpl, jsonResponse,
|
||||
Lists.newArrayList("$.metadata.name", "$.status.containerStatuses[0].restartCount"), http, builder, 100L);
|
||||
|
||||
assertEquals(3, capturedRows.size());
|
||||
assertEquals("pod-a", capturedRows.get(0).getColumns(0));
|
||||
assertEquals("5", capturedRows.get(0).getColumns(1));
|
||||
assertEquals("pod-b-pending", capturedRows.get(1).getColumns(0));
|
||||
assertEquals(CommonConstants.NULL_VALUE, capturedRows.get(1).getColumns(1));
|
||||
assertEquals("pod-c", capturedRows.get(2).getColumns(0));
|
||||
assertEquals("2", capturedRows.get(2).getColumns(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParsePromQlLabelValue() throws Exception {
|
||||
// Create Prometheus format test data
|
||||
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.mqtt;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import java.io.StringWriter;
|
||||
import java.math.BigInteger;
|
||||
import java.security.KeyPair;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.Date;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.MqttProtocol;
|
||||
import org.bouncycastle.asn1.x500.X500Name;
|
||||
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
|
||||
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
|
||||
import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
|
||||
import org.bouncycastle.openssl.jcajce.JcaPKCS8Generator;
|
||||
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class MqttSslFactoryTest {
|
||||
|
||||
private static String certPem;
|
||||
private static String pkcs1KeyPem;
|
||||
private static String pkcs8KeyPem;
|
||||
|
||||
@BeforeAll
|
||||
static void generateCertAndKeys() throws Exception {
|
||||
KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
|
||||
generator.initialize(2048);
|
||||
KeyPair keyPair = generator.generateKeyPair();
|
||||
X500Name subject = new X500Name("CN=hb-3540-mqtt");
|
||||
JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
|
||||
subject, BigInteger.ONE,
|
||||
new Date(System.currentTimeMillis() - 60_000),
|
||||
new Date(System.currentTimeMillis() + 3_600_000),
|
||||
subject, keyPair.getPublic());
|
||||
X509Certificate cert = new JcaX509CertificateConverter()
|
||||
.getCertificate(certBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate())));
|
||||
|
||||
certPem = writePem(cert);
|
||||
pkcs1KeyPem = writePem(keyPair.getPrivate());
|
||||
pkcs8KeyPem = writePem(new JcaPKCS8Generator(keyPair.getPrivate(), null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesPkcs1ClientKey() {
|
||||
assertNotNull(MqttSslFactory.getMslSocketFactory(mqttProtocol(pkcs1KeyPem), true));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parsesPkcs8ClientKey() {
|
||||
assertNotNull(MqttSslFactory.getMslSocketFactory(mqttProtocol(pkcs8KeyPem), true));
|
||||
}
|
||||
|
||||
private static MqttProtocol mqttProtocol(String clientKey) {
|
||||
return MqttProtocol.builder()
|
||||
.tlsVersion("TLSv1.2")
|
||||
.clientCert(certPem)
|
||||
.clientKey(clientKey)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static String writePem(Object object) throws Exception {
|
||||
StringWriter out = new StringWriter();
|
||||
try (JcaPEMWriter writer = new JcaPEMWriter(out)) {
|
||||
writer.writeObject(object);
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
}
|
||||
+25
@@ -30,6 +30,8 @@ import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
class OnlineParserTest {
|
||||
@@ -459,4 +461,27 @@ class OnlineParserTest {
|
||||
assertEquals("run_as", metricFamily.getMetricList().get(0).getLabels().get(3).getName());
|
||||
assertEquals("NT AUTHORITY\nLocalService", metricFamily.getMetricList().get(0).getLabels().get(3).getValue());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseMetricsStopsBeforeSampleBeyondLimit() {
|
||||
final String metrics = "metric_a 1\nmetric_b 2\nmetric_c 3\nmetric_d 4\n";
|
||||
final ByteArrayInputStream inputStream =
|
||||
new ByteArrayInputStream(metrics.getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
assertThrows(OnlineParser.SampleLimitExceededException.class,
|
||||
() -> OnlineParser.parseMetrics(inputStream, 2));
|
||||
|
||||
assertTrue(inputStream.available() > 0, "samples after the limit should remain unread");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testParseMetricsAllowsExactlyTheSampleLimit() throws Exception {
|
||||
final String metrics = "metric_a 1\nmetric_b 2\n";
|
||||
final InputStream inputStream = new ByteArrayInputStream(metrics.getBytes(StandardCharsets.UTF_8));
|
||||
|
||||
final Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream, 2);
|
||||
|
||||
assertNotNull(metricFamilyMap);
|
||||
assertEquals(2, metricFamilyMap.size());
|
||||
}
|
||||
}
|
||||
|
||||
+82
@@ -19,9 +19,12 @@ package org.apache.hertzbeat.collector.collect.script;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.ScriptProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
@@ -138,6 +141,85 @@ public class ScriptCollectImplTest {
|
||||
scriptCollect.collect(builder, metrics);
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
});
|
||||
|
||||
// empty stdout without stderr should be treated as empty one-row data
|
||||
assertDoesNotThrow(() -> {
|
||||
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
|
||||
.charset("utf-8")
|
||||
.parseType("oneRow")
|
||||
.scriptTool("bash")
|
||||
.scriptCommand("grep -o 'centos-hermitlv' /dev/null")
|
||||
.build();
|
||||
Metrics metrics = new Metrics();
|
||||
metrics.setScript(scriptProtocol);
|
||||
metrics.setAliasFields(List.of("nfs_mount"));
|
||||
|
||||
builder = CollectRep.MetricsData.newBuilder();
|
||||
scriptCollect.collect(builder, metrics);
|
||||
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
|
||||
});
|
||||
|
||||
// partial output missing more than one trailing field: the old length check
|
||||
// (lines + 1 < aliases) dropped the whole row here, losing the collected values
|
||||
assertDoesNotThrow(() -> {
|
||||
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
|
||||
.charset("utf-8")
|
||||
.parseType("oneRow")
|
||||
.scriptTool("bash")
|
||||
.scriptCommand("echo 52; echo 35.8033; grep -o 'centos-hermitlv' /dev/null")
|
||||
.build();
|
||||
Metrics metrics = new Metrics();
|
||||
metrics.setScript(scriptProtocol);
|
||||
metrics.setAliasFields(List.of("cpu", "memory", "disk", "nfs_mount"));
|
||||
|
||||
builder = CollectRep.MetricsData.newBuilder();
|
||||
scriptCollect.collect(builder, metrics);
|
||||
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("52", builder.getValues(0).getColumns(0));
|
||||
assertEquals("35.8033", builder.getValues(0).getColumns(1));
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(2));
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
|
||||
});
|
||||
|
||||
// a command that silently exits 1 with no output is indistinguishable from a
|
||||
// grep no-match, so it is deliberately accepted as an empty success
|
||||
assertDoesNotThrow(() -> {
|
||||
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
|
||||
.charset("utf-8")
|
||||
.parseType("oneRow")
|
||||
.scriptTool("bash")
|
||||
.scriptCommand("exit 1")
|
||||
.build();
|
||||
Metrics metrics = new Metrics();
|
||||
metrics.setScript(scriptProtocol);
|
||||
metrics.setAliasFields(List.of("nfs_mount"));
|
||||
|
||||
builder = CollectRep.MetricsData.newBuilder();
|
||||
scriptCollect.collect(builder, metrics);
|
||||
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
|
||||
});
|
||||
|
||||
// non-empty exit code without stderr should still fail when it is not the grep-style no-match case
|
||||
assertDoesNotThrow(() -> {
|
||||
ScriptProtocol scriptProtocol = ScriptProtocol.builder()
|
||||
.charset("utf-8")
|
||||
.parseType("oneRow")
|
||||
.scriptTool("bash")
|
||||
.scriptCommand("exit 2")
|
||||
.build();
|
||||
Metrics metrics = new Metrics();
|
||||
metrics.setScript(scriptProtocol);
|
||||
metrics.setAliasFields(List.of("nfs_mount"));
|
||||
|
||||
builder = CollectRep.MetricsData.newBuilder();
|
||||
scriptCollect.collect(builder, metrics);
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
assertTrue(builder.getMsg().contains("code: 2"));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+136
@@ -22,6 +22,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyBoolean;
|
||||
import static org.mockito.ArgumentMatchers.anyInt;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -30,14 +35,21 @@ import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InterruptedIOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.apache.hertzbeat.collector.collect.common.ssh.SshHelper;
|
||||
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.SshProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.sshd.client.channel.ChannelExec;
|
||||
import org.apache.sshd.client.channel.ClientChannel;
|
||||
import org.apache.sshd.client.channel.ClientChannelEvent;
|
||||
import org.apache.sshd.client.future.OpenFuture;
|
||||
import org.apache.sshd.client.session.ClientSession;
|
||||
import org.apache.sshd.common.SshException;
|
||||
@@ -229,6 +241,130 @@ class SshCollectImplTest {
|
||||
verify(clientSession).close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectPadsPartialOneRowOutput() throws Exception {
|
||||
ChannelExec channel = oneRowChannel("52\n35.8033\n5%", "", 0);
|
||||
Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build();
|
||||
metrics.setAliasFields(List.of("cpu", "memory", "disk", "nfs_mount"));
|
||||
|
||||
ClientSession clientSession = channelSession(channel);
|
||||
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
|
||||
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
|
||||
.thenReturn(clientSession);
|
||||
sshCollect.collect(builder, metrics);
|
||||
}
|
||||
|
||||
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("52", builder.getValues(0).getColumns(0));
|
||||
assertEquals("35.8033", builder.getValues(0).getColumns(1));
|
||||
assertEquals("5%", builder.getValues(0).getColumns(2));
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectTreatsSilentEmptyOneRowOutputAsEmptyRow() throws Exception {
|
||||
ChannelExec channel = oneRowChannel("", "", 1);
|
||||
Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build();
|
||||
metrics.setAliasFields(List.of("nfs_mount"));
|
||||
|
||||
ClientSession clientSession = channelSession(channel);
|
||||
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
|
||||
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
|
||||
.thenReturn(clientSession);
|
||||
sshCollect.collect(builder, metrics);
|
||||
}
|
||||
|
||||
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectFailsOnEmptyOutputWithStderr() throws Exception {
|
||||
ChannelExec channel = oneRowChannel("", "boom: permission denied", 1);
|
||||
Metrics metrics = Metrics.builder().ssh(oneRowProtocol()).build();
|
||||
metrics.setAliasFields(List.of("nfs_mount"));
|
||||
|
||||
ClientSession clientSession = channelSession(channel);
|
||||
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
|
||||
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
|
||||
.thenReturn(clientSession);
|
||||
sshCollect.collect(builder, metrics);
|
||||
}
|
||||
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
assertEquals("boom: permission denied", builder.getMsg());
|
||||
}
|
||||
|
||||
@Test
|
||||
void collectDecodesOutputWithConfiguredCharset() throws Exception {
|
||||
ChannelExec channel = oneRowChannel("挂载正常".getBytes(Charset.forName("GBK")), new byte[0], 0);
|
||||
SshProtocol protocol = oneRowProtocol();
|
||||
protocol.setCharset("GBK");
|
||||
Metrics metrics = Metrics.builder().ssh(protocol).build();
|
||||
metrics.setAliasFields(List.of("nfs_mount"));
|
||||
|
||||
ClientSession clientSession = channelSession(channel);
|
||||
try (MockedStatic<SshHelper> sshHelper = mockStatic(SshHelper.class)) {
|
||||
sshHelper.when(() -> SshHelper.getConnectSession(any(), anyInt(), anyBoolean(), anyBoolean()))
|
||||
.thenReturn(clientSession);
|
||||
sshCollect.collect(builder, metrics);
|
||||
}
|
||||
|
||||
assertEquals(CollectRep.Code.SUCCESS, builder.getCode());
|
||||
assertEquals("挂载正常", builder.getValues(0).getColumns(0));
|
||||
}
|
||||
|
||||
private SshProtocol oneRowProtocol() {
|
||||
return SshProtocol.builder()
|
||||
.host("target.example.com")
|
||||
.port("22")
|
||||
.username("root")
|
||||
.password("password")
|
||||
.timeout("1000")
|
||||
.reuseConnection("true")
|
||||
.useProxy("false")
|
||||
.script("echo ok")
|
||||
.parseType("oneRow")
|
||||
.build();
|
||||
}
|
||||
|
||||
private ClientSession channelSession(ChannelExec channel) throws IOException {
|
||||
ClientSession clientSession = mock(ClientSession.class);
|
||||
when(clientSession.createExecChannel("echo ok")).thenReturn(channel);
|
||||
return clientSession;
|
||||
}
|
||||
|
||||
private ChannelExec oneRowChannel(String stdout, String stderr, int exitStatus) throws IOException {
|
||||
return oneRowChannel(stdout.getBytes(StandardCharsets.UTF_8), stderr.getBytes(StandardCharsets.UTF_8), exitStatus);
|
||||
}
|
||||
|
||||
private ChannelExec oneRowChannel(byte[] stdout, byte[] stderr, int exitStatus) throws IOException {
|
||||
ChannelExec channel = mock(ChannelExec.class);
|
||||
OpenFuture openFuture = mock(OpenFuture.class);
|
||||
CloseFuture closeFuture = mock(CloseFuture.class);
|
||||
AtomicReference<OutputStream> out = new AtomicReference<>();
|
||||
AtomicReference<OutputStream> err = new AtomicReference<>();
|
||||
doAnswer(inv -> {
|
||||
out.set(inv.getArgument(0));
|
||||
return null;
|
||||
}).when(channel).setOut(any());
|
||||
doAnswer(inv -> {
|
||||
err.set(inv.getArgument(0));
|
||||
return null;
|
||||
}).when(channel).setErr(any());
|
||||
when(channel.open()).thenReturn(openFuture);
|
||||
when(channel.waitFor(any(), anyLong())).thenAnswer(inv -> {
|
||||
out.get().write(stdout);
|
||||
err.get().write(stderr);
|
||||
return Set.of(ClientChannelEvent.CLOSED);
|
||||
});
|
||||
when(channel.getExitStatus()).thenReturn(exitStatus);
|
||||
when(channel.close(false)).thenReturn(closeFuture);
|
||||
when(closeFuture.await(anyLong())).thenReturn(true);
|
||||
return channel;
|
||||
}
|
||||
|
||||
private SshProtocol protocol(int timeout) {
|
||||
return SshProtocol.builder()
|
||||
.host("target.example.com")
|
||||
|
||||
+110
@@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
@@ -30,6 +31,7 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.apache.commons.net.telnet.TelnetClient;
|
||||
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.TelnetProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
@@ -155,6 +157,114 @@ class TelnetCollectImplTest {
|
||||
mocked.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectPadsMissingMetrics() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
Metrics metrics = telnetMetrics("mntr", List.of("responseTime", "a", "b", "c"));
|
||||
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("a=1")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("1", builder.getValues(0).getColumns(1));
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(2));
|
||||
assertEquals(CommonConstants.NULL_VALUE, builder.getValues(0).getColumns(3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectFailsWithRawReplyWhenNothingParsed() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
Metrics metrics = telnetMetrics("conf", List.of("responseTime", "a"));
|
||||
try (MockedConstruction<TelnetClient> mocked =
|
||||
mockTelnetReply("conf is not executed because it is not in the whitelist.")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
assertTrue(builder.getMsg().contains("not in the whitelist"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectKeepsFirstOnDuplicateKeys() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
Metrics metrics = telnetMetrics("mntr", List.of("a", "b"));
|
||||
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("a=1\na=2\nb=3")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("1", builder.getValues(0).getColumns(0));
|
||||
assertEquals("3", builder.getValues(0).getColumns(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectFailsWhenReplyHasOnlyUnrelatedPairs() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
Metrics metrics = telnetMetrics("mntr", List.of("responseTime", "a"));
|
||||
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("error=conf disabled")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectSurvivesHeaderOnlyEnviReply() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder().setApp("zookeeper");
|
||||
Metrics metrics = telnetMetrics("envi", List.of("responseTime", "a"));
|
||||
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("Environment:")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectFailsCleanlyOnNewlineOnlyReply() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder().setApp("zookeeper");
|
||||
Metrics metrics = telnetMetrics("conf", List.of("responseTime", "a"));
|
||||
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("\n")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(CollectRep.Code.FAIL, builder.getCode());
|
||||
assertTrue(builder.getMsg().contains("returned no expected metrics"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectKeepsValueContainingSeparator() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
Metrics metrics = telnetMetrics("conf", List.of("dataDir", "secureClientPort"));
|
||||
try (MockedConstruction<TelnetClient> mocked = mockTelnetReply("dataDir=/data/zk=a\nsecureClientPort=")) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
assertEquals("/data/zk=a", builder.getValues(0).getColumns(0));
|
||||
assertEquals("", builder.getValues(0).getColumns(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCollectBlankCmdKeepsResponseTimeRow() {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
Metrics metrics = telnetMetrics("", List.of("responseTime"));
|
||||
try (MockedConstruction<TelnetClient> mocked =
|
||||
Mockito.mockConstruction(TelnetClient.class, (telnetClient, context) ->
|
||||
Mockito.when(telnetClient.isConnected()).thenReturn(true))) {
|
||||
telnetCollect.collect(builder, metrics);
|
||||
}
|
||||
assertEquals(1, builder.getValuesCount());
|
||||
}
|
||||
|
||||
private static Metrics telnetMetrics(String cmd, List<String> aliasFields) {
|
||||
Metrics metrics = new Metrics();
|
||||
metrics.setTelnet(TelnetProtocol.builder().timeout("10").port("2181").cmd(cmd).build());
|
||||
metrics.setAliasFields(aliasFields);
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private static MockedConstruction<TelnetClient> mockTelnetReply(String reply) {
|
||||
InputStream inputStream = new ByteArrayInputStream(reply.getBytes(StandardCharsets.UTF_8));
|
||||
return Mockito.mockConstruction(TelnetClient.class, (telnetClient, context) -> {
|
||||
Mockito.when(telnetClient.isConnected()).thenReturn(true);
|
||||
Mockito.when(telnetClient.getOutputStream()).thenReturn(Mockito.mock(OutputStream.class));
|
||||
Mockito.when(telnetClient.getInputStream()).thenReturn(inputStream);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void preCheck() throws IllegalArgumentException {
|
||||
// metrics is null
|
||||
|
||||
+9
-3
@@ -252,11 +252,12 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
|
||||
if (metrics.getCalculates() == null) {
|
||||
metrics.setCalculates(Collections.emptyList());
|
||||
}
|
||||
List<String> aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList);
|
||||
// eg: database_pages=Database pages unconventional mapping
|
||||
Map<String, String> fieldAliasMap = new HashMap<>(8);
|
||||
Map<String, JexlExpression> fieldExpressionMap = metrics.getCalculates()
|
||||
.stream()
|
||||
.map(cal -> transformCal(cal, fieldAliasMap))
|
||||
.map(cal -> transformCal(cal, fieldAliasMap, aliasFields))
|
||||
.filter(Objects::nonNull)
|
||||
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (JexlExpression) arr[1], (oldValue, newValue) -> newValue));
|
||||
|
||||
@@ -270,7 +271,6 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
|
||||
.collect(Collectors.toMap(arr -> (String) arr[0], arr -> (Pair<String, String>) arr[1], (oldValue, newValue) -> newValue));
|
||||
|
||||
List<Metrics.Field> fields = metrics.getFields();
|
||||
List<String> aliasFields = Optional.ofNullable(metrics.getAliasFields()).orElseGet(Collections::emptyList);
|
||||
Map<String, String> aliasFieldValueMap = new HashMap<>(8);
|
||||
Map<String, Object> fieldValueMap = new HashMap<>(8);
|
||||
Map<String, Object> stringTypefieldValueMap = new HashMap<>(8);
|
||||
@@ -420,13 +420,19 @@ public class MetricsCollect implements Runnable, Comparable<MetricsCollect> {
|
||||
* @param fieldAliasMap field alias map
|
||||
* @return expr
|
||||
*/
|
||||
private Object[] transformCal(String cal, Map<String, String> fieldAliasMap) {
|
||||
private Object[] transformCal(String cal, Map<String, String> fieldAliasMap, List<String> aliasFields) {
|
||||
int splitIndex = cal.indexOf("=");
|
||||
if (splitIndex < 0) {
|
||||
return null;
|
||||
}
|
||||
String field = cal.substring(0, splitIndex).trim();
|
||||
String expressionStr = cal.substring(splitIndex + 1).trim().replace("\\#", "#");
|
||||
// a direct alias reference (RHS must exactly equal an aliasField, no whitespace/case tolerance) is not a formula,
|
||||
// JEXL parses "[0]" in such paths as array access and silently returns null
|
||||
if (aliasFields.contains(expressionStr)) {
|
||||
fieldAliasMap.put(field, expressionStr);
|
||||
return null;
|
||||
}
|
||||
JexlExpression expression;
|
||||
try {
|
||||
expression = JexlExpressionRunner.compile(expressionStr);
|
||||
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.dispatch;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.timer.Timeout;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Test case for {@link MetricsCollect}
|
||||
*/
|
||||
class MetricsCollectTest {
|
||||
|
||||
@Test
|
||||
void calculateFieldsMapsIndexedJsonPathAlias() {
|
||||
Metrics metrics = Metrics.builder()
|
||||
.name("pods")
|
||||
.priority((byte) 0)
|
||||
.fields(List.of(
|
||||
Metrics.Field.builder().field("pod").type(CommonConstants.TYPE_STRING).build(),
|
||||
Metrics.Field.builder().field("rc").type(CommonConstants.TYPE_STRING).build()))
|
||||
.aliasFields(List.of("$.metadata.name", "$.status.containerStatuses[0].restartCount"))
|
||||
.calculates(List.of(
|
||||
"pod=$.metadata.name",
|
||||
"rc=$.status.containerStatuses[0].restartCount"))
|
||||
.build();
|
||||
|
||||
Timeout timeout = mock(Timeout.class);
|
||||
WheelTimerTask timerTask = mock(WheelTimerTask.class);
|
||||
when(timeout.task()).thenReturn(timerTask);
|
||||
when(timerTask.getJob()).thenReturn(Job.builder().build());
|
||||
MetricsCollect metricsCollect = new MetricsCollect(metrics, timeout, null, "test", List.of());
|
||||
|
||||
CollectRep.MetricsData.Builder collectData = CollectRep.MetricsData.newBuilder();
|
||||
collectData.addValueRow(CollectRep.ValueRow.newBuilder()
|
||||
.addColumn("pod-a").addColumn("5").build());
|
||||
collectData.addValueRow(CollectRep.ValueRow.newBuilder()
|
||||
.addColumn("pod-b-pending").addColumn(CommonConstants.NULL_VALUE).build());
|
||||
|
||||
metricsCollect.calculateFields(metrics, collectData);
|
||||
|
||||
List<CollectRep.ValueRow> rows = collectData.getValuesList();
|
||||
assertEquals(2, rows.size());
|
||||
assertEquals("pod-a", rows.get(0).getColumns(0));
|
||||
assertEquals("5", rows.get(0).getColumns(1));
|
||||
assertEquals("pod-b-pending", rows.get(1).getColumns(0));
|
||||
assertEquals(CommonConstants.NULL_VALUE, rows.get(1).getColumns(1));
|
||||
}
|
||||
}
|
||||
+18
@@ -36,12 +36,16 @@ public final class JsonPathParser {
|
||||
|
||||
private static final ParseContext PARSER;
|
||||
|
||||
private static final ParseContext ROW_PARSER;
|
||||
|
||||
static {
|
||||
Configuration conf = Configuration.defaultConfiguration()
|
||||
.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL)
|
||||
.addOptions(Option.ALWAYS_RETURN_LIST);
|
||||
CacheProvider.setCache(new LRUCache(128));
|
||||
PARSER = JsonPath.using(conf);
|
||||
// a single row legitimately may not contain the queried path
|
||||
ROW_PARSER = JsonPath.using(conf.addOptions(Option.SUPPRESS_EXCEPTIONS));
|
||||
}
|
||||
|
||||
private JsonPathParser() {
|
||||
@@ -73,4 +77,18 @@ public final class JsonPathParser {
|
||||
return PARSER.parse(content).read(jsonPath, typeRef);
|
||||
}
|
||||
|
||||
/**
|
||||
* use json path to parse one already-parsed row object, missing paths yield an empty list
|
||||
* @param document parsed json object of a single row
|
||||
* @param jsonPath jsonPath relative to the row root
|
||||
* @return matched values, empty list when the path does not exist in this row
|
||||
*/
|
||||
public static List<Object> parseRowWithJsonPath(Object document, String jsonPath) {
|
||||
if (document == null || StringUtils.isEmpty(jsonPath)) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<Object> values = ROW_PARSER.parse(document).read(jsonPath);
|
||||
return values == null ? Collections.emptyList() : values;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.util;
|
||||
|
||||
import com.jayway.jsonpath.PathNotFoundException;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Test case for {@link JsonPathParser}
|
||||
*/
|
||||
class JsonPathParserTest {
|
||||
|
||||
private static final String ROW_JSON = "{\"metadata\": {\"name\": \"pod-a\"},"
|
||||
+ " \"status\": {\"phase\": \"Running\","
|
||||
+ " \"containerStatuses\": [{\"name\": \"c1\", \"ready\": true, \"restartCount\": 5}]}}";
|
||||
|
||||
private Object row() {
|
||||
return JsonPathParser.parseContentWithJsonPath(ROW_JSON, "$").get(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseRowWithJsonPathReturnsExistingValue() {
|
||||
List<Object> values = JsonPathParser.parseRowWithJsonPath(row(), "$.status.containerStatuses[0].restartCount");
|
||||
|
||||
assertEquals(1, values.size());
|
||||
assertEquals(5, values.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseRowWithJsonPathReturnsEmptyListWhenPathMissing() {
|
||||
Object pendingRow = JsonPathParser
|
||||
.parseContentWithJsonPath("{\"metadata\": {\"name\": \"pod-b\"}, \"status\": {\"phase\": \"Pending\"}}", "$")
|
||||
.get(0);
|
||||
|
||||
List<Object> values = JsonPathParser.parseRowWithJsonPath(pendingRow, "$.status.containerStatuses[0].restartCount");
|
||||
|
||||
assertTrue(values.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseRowWithJsonPathReturnsAllValuesForWildcard() {
|
||||
List<Object> values = JsonPathParser.parseRowWithJsonPath(row(), "$.status.containerStatuses[0].*");
|
||||
|
||||
assertEquals(3, values.size());
|
||||
assertTrue(values.contains("c1"));
|
||||
assertTrue(values.contains(true));
|
||||
assertTrue(values.contains(5));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseContentWithJsonPathStillThrowsWhenPathMissing() {
|
||||
assertThrows(PathNotFoundException.class,
|
||||
() -> JsonPathParser.parseContentWithJsonPath(ROW_JSON, "$.spec.nodeName"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseRowWithJsonPathHandlesNullDocumentAndEmptyPath() {
|
||||
assertTrue(JsonPathParser.parseRowWithJsonPath(null, "$.status").isEmpty());
|
||||
assertTrue(JsonPathParser.parseRowWithJsonPath(row(), "").isEmpty());
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,20 @@
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcpkix-jdk15on</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcprov-jdk15on</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcpkix-jdk18on</artifactId>
|
||||
<version>${bouncycastle.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.nebulagraph;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
import com.vesoft.nebula.client.graph.data.CASignedSSLParam;
|
||||
import com.vesoft.nebula.util.SslUtil;
|
||||
import java.io.FileWriter;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.file.Path;
|
||||
import java.security.KeyPair;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.Date;
|
||||
import org.bouncycastle.asn1.x500.X500Name;
|
||||
import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
|
||||
import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder;
|
||||
import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
|
||||
import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
class VesoftSslBouncyCastleSmokeTest {
|
||||
|
||||
@Test
|
||||
void vesoftSslUtilWorksWithBouncyCastleJdk18on(@TempDir Path dir) throws Exception {
|
||||
KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA");
|
||||
generator.initialize(2048);
|
||||
KeyPair keyPair = generator.generateKeyPair();
|
||||
X500Name subject = new X500Name("CN=hb-3540-smoke");
|
||||
JcaX509v3CertificateBuilder certBuilder = new JcaX509v3CertificateBuilder(
|
||||
subject, BigInteger.ONE,
|
||||
new Date(System.currentTimeMillis() - 60_000),
|
||||
new Date(System.currentTimeMillis() + 3_600_000),
|
||||
subject, keyPair.getPublic());
|
||||
X509Certificate cert = new JcaX509CertificateConverter()
|
||||
.getCertificate(certBuilder.build(new JcaContentSignerBuilder("SHA256withRSA").build(keyPair.getPrivate())));
|
||||
|
||||
Path crt = dir.resolve("smoke.crt");
|
||||
Path key = dir.resolve("smoke.key");
|
||||
try (JcaPEMWriter writer = new JcaPEMWriter(new FileWriter(crt.toFile()))) {
|
||||
writer.writeObject(cert);
|
||||
}
|
||||
try (JcaPEMWriter writer = new JcaPEMWriter(new FileWriter(key.toFile()))) {
|
||||
writer.writeObject(keyPair.getPrivate());
|
||||
}
|
||||
|
||||
CASignedSSLParam param = new CASignedSSLParam(crt.toString(), crt.toString(), key.toString());
|
||||
assertNotNull(SslUtil.getSSLSocketFactoryWithCA(param));
|
||||
}
|
||||
}
|
||||
+5
@@ -86,6 +86,11 @@ public class SshProtocol implements CommonRequestProtocol, Protocol {
|
||||
*/
|
||||
private String parseType;
|
||||
|
||||
/**
|
||||
* Charset of the remote command output, default UTF-8
|
||||
*/
|
||||
private String charset;
|
||||
|
||||
/**
|
||||
* IP ADDRESS OR DOMAIN NAME OF THE PEER PROXY HOST
|
||||
*/
|
||||
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.http;
|
||||
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.http.HttpCollectImpl;
|
||||
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
|
||||
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
|
||||
import org.apache.hertzbeat.collector.dispatch.unit.impl.DataSizeConvert;
|
||||
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
|
||||
import org.apache.hertzbeat.collector.util.CollectUtil;
|
||||
import org.apache.hertzbeat.common.entity.job.Configmap;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.timer.Timeout;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Integration test for etcd monitoring functionality.
|
||||
* Fixture at src/test/resources/http/etcd/metrics.txt is a real capture from a live
|
||||
* etcd v3.5.17 /metrics endpoint (see app-etcd.yml for the corresponding template).
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
public class EtcdMonitorE2eTest extends AbstractCollectE2eTest {
|
||||
|
||||
private static final int MOCK_SERVER_PORT = 52379;
|
||||
private static final String LOCALHOST = "127.0.0.1";
|
||||
private static HttpServer mockServer;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() {
|
||||
if (mockServer != null) {
|
||||
mockServer.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
collect = new HttpCollectImpl();
|
||||
|
||||
// the shared harness wires MetricsCollect with an empty unit-convert list,
|
||||
// but this template relies on B->MB conversion
|
||||
Timeout convertTimeout = mock(Timeout.class);
|
||||
WheelTimerTask convertTimerJob = mock(WheelTimerTask.class);
|
||||
when(convertTimeout.task()).thenReturn(convertTimerJob);
|
||||
when(convertTimerJob.getJob()).thenReturn(mock(Job.class));
|
||||
metricsCollect = new MetricsCollect(mock(Metrics.class), convertTimeout,
|
||||
mock(CollectDataDispatch.class), null, List.of(new DataSizeConvert()));
|
||||
|
||||
String metricsResponse = loadResponseFromFile("classpath:http/etcd/metrics.txt");
|
||||
|
||||
mockServer = HttpServer.create(new InetSocketAddress(MOCK_SERVER_PORT), 0);
|
||||
mockServer.setExecutor(null);
|
||||
mockServer.start();
|
||||
mockServer.createContext("/metrics", exchange -> sendTextResponse(exchange, metricsResponse));
|
||||
}
|
||||
|
||||
private String loadResponseFromFile(String resourcePath) throws Exception {
|
||||
return new String(Files.readAllBytes(ResourceUtils.getFile(resourcePath).toPath()));
|
||||
}
|
||||
|
||||
private void sendTextResponse(HttpExchange exchange, String response) throws IOException {
|
||||
exchange.getResponseHeaders().set("Content-Type", "text/plain");
|
||||
final byte[] array = response.getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(200, array.length);
|
||||
try (OutputStream os = exchange.getResponseBody()) {
|
||||
os.write(array);
|
||||
}
|
||||
}
|
||||
|
||||
private static final Map<String, String> EXPECTED_VALUES = Map.of(
|
||||
"etcd_server_has_leader", "1",
|
||||
"etcd_mvcc_db_total_size_in_bytes", "0.0195",
|
||||
"etcd_server_leader_changes_seen_total", "1",
|
||||
"process_cpu_seconds_total", "42.41",
|
||||
"process_resident_memory_bytes", "29.9844");
|
||||
|
||||
@Test
|
||||
public void testEtcdMonitor() {
|
||||
Job etcdJob = appService.getAppDefine("etcd");
|
||||
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
|
||||
for (Metrics metricsDef : etcdJob.getMetrics()) {
|
||||
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef,
|
||||
!configmapFromPreCollectData.isEmpty() ? configmapFromPreCollectData.get(0) : new HashMap<>());
|
||||
CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricsDef.getName());
|
||||
Assertions.assertEquals(EXPECTED_VALUES.get(metricsDef.getName()),
|
||||
metricsData.getValues().get(0).getColumns(0),
|
||||
metricsDef.getName() + " collected value mismatch");
|
||||
configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
HttpProtocol protocol = new HttpProtocol();
|
||||
protocol.setHost(LOCALHOST);
|
||||
protocol.setPort(String.valueOf(MOCK_SERVER_PORT));
|
||||
protocol.setMethod(metricsDef.getHttp().getMethod());
|
||||
protocol.setParseType(metricsDef.getHttp().getParseType());
|
||||
protocol.setParseScript(metricsDef.getHttp().getParseScript());
|
||||
protocol.setUrl(metricsDef.getHttp().getUrl());
|
||||
return protocol;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
HttpProtocol protocol = (HttpProtocol) buildProtocol(metricsDef);
|
||||
metrics.setHttp(protocol);
|
||||
// prometheus parseType filters by builder.getMetrics(); production sets it in
|
||||
// MetricsCollect.run() but this test harness does not, so set it here
|
||||
CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder()
|
||||
.setMetrics(metricsDef.getName());
|
||||
return collectMetricsData(metrics, metricsDef, metricsData);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Captured from a live etcd v3.5.17 /metrics endpoint, trimmed to the metric
|
||||
# families used by app-etcd.yml.
|
||||
# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes.
|
||||
# TYPE etcd_mvcc_db_total_size_in_bytes gauge
|
||||
etcd_mvcc_db_total_size_in_bytes 20480
|
||||
# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not.
|
||||
# TYPE etcd_server_has_leader gauge
|
||||
etcd_server_has_leader 1
|
||||
# HELP etcd_server_leader_changes_seen_total The number of leader changes seen.
|
||||
# TYPE etcd_server_leader_changes_seen_total counter
|
||||
etcd_server_leader_changes_seen_total 1
|
||||
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
|
||||
# TYPE process_cpu_seconds_total counter
|
||||
process_cpu_seconds_total 42.41
|
||||
# HELP process_resident_memory_bytes Resident memory size in bytes.
|
||||
# TYPE process_resident_memory_bytes gauge
|
||||
process_resident_memory_bytes 3.1440896e+07
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -86,9 +86,16 @@ resourceRole:
|
||||
- /api/ingestion/otlp/**===get===[admin,user,guest]
|
||||
- /api/logs/**===get===[admin,user,guest]
|
||||
- /api/traces/**===get===[admin,user,guest]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -121,10 +128,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.config;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import org.springdoc.core.properties.SwaggerUiConfigProperties;
|
||||
import org.springdoc.core.properties.SwaggerUiOAuthProperties;
|
||||
import org.springdoc.core.providers.ObjectMapperProvider;
|
||||
import org.springdoc.webmvc.ui.SwaggerIndexPageTransformer;
|
||||
import org.springdoc.webmvc.ui.SwaggerWelcomeCommon;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.web.servlet.resource.ResourceTransformerChain;
|
||||
import org.springframework.web.servlet.resource.TransformedResource;
|
||||
|
||||
/**
|
||||
* Adds the HertzBeat access token to same-origin requests made by Swagger UI.
|
||||
*/
|
||||
final class AuthorizedSwaggerIndexTransformer extends SwaggerIndexPageTransformer {
|
||||
|
||||
private static final String SWAGGER_INITIALIZER = "swagger-initializer.js";
|
||||
|
||||
private static final String PRESETS_MARKER = "presets: [";
|
||||
|
||||
private static final String INTERCEPTOR_MARKER = "requestInterceptor: (request) => {";
|
||||
|
||||
private static final String INTERCEPTOR_RETURN = "return request;";
|
||||
|
||||
private static final String AUTHORIZATION_LOGIC = """
|
||||
const currentUrl = new URL(document.URL);
|
||||
const requestUrl = new URL(request.url, document.location.origin);
|
||||
const sameOrigin = currentUrl.protocol === requestUrl.protocol && currentUrl.host === requestUrl.host;
|
||||
const token = window.localStorage.getItem('Authorization');
|
||||
if (sameOrigin && token) {
|
||||
request.headers['Authorization'] = `Bearer ${token}`;
|
||||
}
|
||||
""";
|
||||
|
||||
private static final String AUTHORIZATION_INTERCEPTOR = """
|
||||
requestInterceptor: (request) => {
|
||||
%s
|
||||
return request;
|
||||
},
|
||||
""".formatted(AUTHORIZATION_LOGIC.stripTrailing());
|
||||
|
||||
AuthorizedSwaggerIndexTransformer(SwaggerUiConfigProperties swaggerUiConfig,
|
||||
SwaggerUiOAuthProperties swaggerUiOauthProperties,
|
||||
SwaggerWelcomeCommon swaggerWelcomeCommon,
|
||||
ObjectMapperProvider objectMapperProvider) {
|
||||
super(swaggerUiConfig, swaggerUiOauthProperties, swaggerWelcomeCommon, objectMapperProvider);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Resource transform(HttpServletRequest request, Resource resource,
|
||||
ResourceTransformerChain transformerChain) throws IOException {
|
||||
final Resource transformed = super.transform(request, resource, transformerChain);
|
||||
if (!SWAGGER_INITIALIZER.equals(resource.getFilename())) {
|
||||
return transformed;
|
||||
}
|
||||
final String initializer;
|
||||
try (final var input = transformed.getInputStream()) {
|
||||
initializer = new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
return new TransformedResource(transformed,
|
||||
addAuthorizationInterceptor(initializer).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
/**
|
||||
* Springdoc writes an interceptor of its own when csrf support is turned on. Two keys
|
||||
* of the same name would silently drop one of them, so the token is appended to the
|
||||
* body already there. It goes in at the end because the last write to a header wins,
|
||||
* and springdoc can be configured to write to {@code Authorization} as well. The
|
||||
* declarations above are named apart from the ones springdoc emits on purpose: the
|
||||
* two bodies share a scope, so a collision would be a syntax error.
|
||||
*
|
||||
* @param initializer the swagger initializer script
|
||||
* @return the script with the token attached to its outgoing requests
|
||||
*/
|
||||
static String addAuthorizationInterceptor(String initializer) {
|
||||
final int interceptor = initializer.indexOf(INTERCEPTOR_MARKER);
|
||||
if (interceptor >= 0) {
|
||||
final int returnStatement = initializer.indexOf(INTERCEPTOR_RETURN,
|
||||
interceptor + INTERCEPTOR_MARKER.length());
|
||||
if (returnStatement < 0) {
|
||||
throw new IllegalStateException("the swagger initializer interceptor no longer returns the request");
|
||||
}
|
||||
return insertLineBefore(initializer, returnStatement, AUTHORIZATION_LOGIC);
|
||||
}
|
||||
final int presets = initializer.indexOf(PRESETS_MARKER);
|
||||
if (presets < 0) {
|
||||
throw new IllegalStateException("the swagger initializer no longer contains the presets marker");
|
||||
}
|
||||
return insertLineBefore(initializer, presets, AUTHORIZATION_INTERCEPTOR);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param initializer the swagger initializer script
|
||||
* @param index an index into the line to insert in front of
|
||||
* @param insertion the lines to insert, newline terminated
|
||||
* @return the script with the insertion on its own lines, leaving the indentation of
|
||||
* the line at {@code index} alone
|
||||
*/
|
||||
private static String insertLineBefore(String initializer, int index, String insertion) {
|
||||
final int lineStart = initializer.lastIndexOf('\n', index) + 1;
|
||||
return initializer.substring(0, lineStart) + insertion + initializer.substring(lineStart);
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -35,7 +35,9 @@ public class SecurityCorsConfiguration {
|
||||
public FilterRegistrationBean corsFilter() {
|
||||
UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource();
|
||||
CorsConfiguration corsConfiguration = new CorsConfiguration();
|
||||
corsConfiguration.setAllowCredentials(true);
|
||||
// Requests authenticate with a token in the Authorization header rather than a
|
||||
// cookie, so no request relies on ambient credentials being sent cross origin.
|
||||
corsConfiguration.setAllowCredentials(false);
|
||||
corsConfiguration.setAllowedOriginPatterns(Collections.singletonList(CorsConfiguration.ALL));
|
||||
corsConfiguration.addAllowedHeader(CorsConfiguration.ALL);
|
||||
corsConfiguration.addAllowedMethod(CorsConfiguration.ALL);
|
||||
|
||||
+28
@@ -25,6 +25,13 @@ import io.swagger.v3.oas.models.info.Info;
|
||||
import io.swagger.v3.oas.models.info.License;
|
||||
import io.swagger.v3.oas.models.security.SecurityRequirement;
|
||||
import io.swagger.v3.oas.models.security.SecurityScheme;
|
||||
import org.springdoc.core.properties.SwaggerUiConfigProperties;
|
||||
import org.springdoc.core.properties.SwaggerUiOAuthProperties;
|
||||
import org.springdoc.core.providers.ObjectMapperProvider;
|
||||
import org.springdoc.webmvc.ui.SwaggerIndexTransformer;
|
||||
import org.springdoc.webmvc.ui.SwaggerWelcomeCommon;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
@@ -37,6 +44,27 @@ public class SwaggerConfig {
|
||||
|
||||
private static final String SECURITY_SCHEME_NAME = "BearerAuth";
|
||||
|
||||
/**
|
||||
* The springdoc beans this one is built from only exist while both switches are on:
|
||||
* its own ui configuration is conditional on {@code SpringDocConfiguration}, which
|
||||
* {@code springdoc.api-docs.enabled} gates in turn. Matching both switches keeps a
|
||||
* deployment that turns the document off from failing to start.
|
||||
*
|
||||
* @return the swagger ui index transformer, replacing the springdoc default
|
||||
*/
|
||||
@Bean
|
||||
@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET)
|
||||
@ConditionalOnProperty(name = {"springdoc.api-docs.enabled", "springdoc.swagger-ui.enabled"},
|
||||
havingValue = "true", matchIfMissing = true)
|
||||
public SwaggerIndexTransformer authorizedSwaggerIndexTransformer(
|
||||
SwaggerUiConfigProperties swaggerUiConfig,
|
||||
SwaggerUiOAuthProperties swaggerUiOauthProperties,
|
||||
SwaggerWelcomeCommon swaggerWelcomeCommon,
|
||||
ObjectMapperProvider objectMapperProvider) {
|
||||
return new AuthorizedSwaggerIndexTransformer(swaggerUiConfig, swaggerUiOauthProperties,
|
||||
swaggerWelcomeCommon, objectMapperProvider);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public OpenAPI springOpenApi() {
|
||||
return new OpenAPI()
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The monitoring type category:service-application service monitoring db-database monitoring custom-custom monitoring os-operating system monitoring mid-middleware
|
||||
category: mid
|
||||
# The monitoring type eg: linux windows tomcat mysql aws...
|
||||
app: etcd
|
||||
# The monitoring i18n name
|
||||
name:
|
||||
zh-CN: etcd
|
||||
en-US: etcd
|
||||
ja-JP: etcd
|
||||
# The description and help of this monitoring type
|
||||
help:
|
||||
zh-CN: HertzBeat 通过调用 <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>etcd Prometheus Metrics 接口</a>(默认在客户端端口 <i>2379</i> 的 <i>/metrics</i> 路径)对 etcd 键值存储(3.4+)的领导者状态、数据库大小、进程资源等指标进行采集监控。<br>您可以点击“<i>新建 etcd</i>”并配置 HOST 端口等相关参数进行添加。<br><span class='help_module_span'>⚠️注意:请确保 HertzBeat 能访问 etcd 的 /metrics 接口。该接口默认由 client listener 提供;若 etcd 仅监听 localhost 或客户端启用了双向 TLS,请通过 --listen-metrics-urls 配置独立的 metrics 地址。</span>
|
||||
en-US: HertzBeat monitors the etcd key-value store's (3.4+) leader status, database size and process resource usage by calling the <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>etcd Prometheus Metrics endpoint</a> (default at the <i>/metrics</i> path on the client port <i>2379</i>).<br>You can click "<i>New etcd</i>" and configure the host, port and other related params to add it.<br><span class='help_module_span'>Note - make sure HertzBeat can reach etcd's /metrics endpoint. It is served on the client listener by default; if etcd only listens on localhost or client mutual TLS is enabled, configure a dedicated metrics address via --listen-metrics-urls.</span>
|
||||
zh-TW: HertzBeat 透過調用 <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>etcd Prometheus Metrics 介面</a>(預設在客戶端連接埠 <i>2379</i> 的 <i>/metrics</i> 路徑)對 etcd 鍵值儲存(3.4+)的領導者狀態、資料庫大小、程序資源等指標進行採集監控。<br>您可以點擊“<i>新建 etcd</i>”並配置 HOST 連接埠等相關參數進行添加。<br><span class='help_module_span'>⚠️注意:請確保 HertzBeat 能訪問 etcd 的 /metrics 介面。該介面預設由 client listener 提供;若 etcd 僅監聽 localhost 或客戶端啟用了雙向 TLS,請透過 --listen-metrics-urls 配置獨立的 metrics 地址。</span>
|
||||
ja-JP: HertzBeat は etcd(3.4+)が公開する <a href='https://etcd.io/docs/latest/metrics/' class='help_module_content'>Prometheus metrics エンドポイント</a>(デフォルトではクライアントポート <i>2379</i> の <i>/metrics</i> パス)から、リーダー状態・データベースサイズ・プロセスリソース等の指標を収集し、etcd キーバリューストアを監視します。<br>「<i>新規 etcd</i>」をクリックしてホストやポートなどのパラメータを設定して追加できます。<br><span class='help_module_span'>⚠️注意:HertzBeat が etcd の /metrics エンドポイントへ到達できることを確認してください。デフォルトでは client listener が提供しますが、etcd が localhost のみを監視している場合やクライアント相互 TLS が有効な場合は、--listen-metrics-urls で専用の metrics アドレスを設定してください。</span>
|
||||
helpLink:
|
||||
zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/etcd/
|
||||
en-US: https://hertzbeat.apache.org/docs/help/etcd/
|
||||
# Input params define for monitoring(render web ui by the definition)
|
||||
params:
|
||||
# field-param field key
|
||||
- field: host
|
||||
# name-param field display i18n name
|
||||
name:
|
||||
zh-CN: 目标Host
|
||||
en-US: Target Host
|
||||
ja-JP: 目標ホスト
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: host
|
||||
# required-true or false
|
||||
required: true
|
||||
- field: port
|
||||
name:
|
||||
zh-CN: 端口
|
||||
en-US: Port
|
||||
ja-JP: ポート
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: number
|
||||
# when type is number, range is required
|
||||
range: '[0,65535]'
|
||||
# default value: etcd client port that serves /metrics
|
||||
defaultValue: 2379
|
||||
required: true
|
||||
- field: timeout
|
||||
name:
|
||||
zh-CN: 查询超时时间
|
||||
en-US: Query Timeout
|
||||
ja-JP: クエリタイムアウト
|
||||
type: number
|
||||
required: false
|
||||
# hide param-true or false
|
||||
hide: true
|
||||
defaultValue: 6000
|
||||
# field-param field key
|
||||
- field: ssl
|
||||
# name-param field display i18n name
|
||||
name:
|
||||
zh-CN: 启用HTTPS
|
||||
en-US: HTTPS
|
||||
ja-JP: HTTPS
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: boolean
|
||||
hide: true
|
||||
# field-param field key
|
||||
- field: headers
|
||||
# name-param field display i18n name
|
||||
name:
|
||||
zh-CN: 请求Headers
|
||||
en-US: Headers
|
||||
ja-JP: ヘッダ
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: key-value
|
||||
# required-true or false
|
||||
required: false
|
||||
hide: true
|
||||
# when type is key-value, use keyAlias to config key alias name
|
||||
keyAlias: Header Name
|
||||
# when type is key-value, use valueAlias to config value alias name
|
||||
valueAlias: Header Value
|
||||
# field-param field key
|
||||
- field: authType
|
||||
# name-param field display i18n name
|
||||
name:
|
||||
zh-CN: 认证方式
|
||||
en-US: Auth Type
|
||||
ja-JP: 認証方法
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: radio
|
||||
# required-true or false
|
||||
required: false
|
||||
# hide param-true or false
|
||||
hide: true
|
||||
# when type is radio checkbox, use option to show optional values {name1:value1,name2:value2}
|
||||
options:
|
||||
- label: Basic Auth
|
||||
value: Basic Auth
|
||||
- label: Digest Auth
|
||||
value: Digest Auth
|
||||
# field-param field key
|
||||
- field: username
|
||||
# name-param field display i18n name
|
||||
name:
|
||||
zh-CN: 用户名
|
||||
en-US: Username
|
||||
ja-JP: ユーザー名
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: text
|
||||
# when type is text, use limit to limit string length
|
||||
limit: 50
|
||||
# required-true or false
|
||||
required: false
|
||||
# hide param-true or false
|
||||
hide: true
|
||||
# field-param field key
|
||||
- field: password
|
||||
# name-param field display i18n name
|
||||
name:
|
||||
zh-CN: 密码
|
||||
en-US: Password
|
||||
ja-JP: パスワード
|
||||
# type-param field type(most mapping the html input type)
|
||||
type: password
|
||||
# required-true or false
|
||||
required: false
|
||||
# hide param-true or false
|
||||
hide: true
|
||||
# collect metrics config list
|
||||
# each metrics group name must exactly match a Prometheus metric family name exposed by etcd /metrics
|
||||
metrics:
|
||||
# metrics - etcd_server_has_leader (availability: whether this etcd member has a raft leader)
|
||||
- name: etcd_server_has_leader
|
||||
i18n:
|
||||
zh-CN: 领导者状态
|
||||
en-US: Leader Status
|
||||
ja-JP: リーダー状態
|
||||
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
|
||||
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
|
||||
priority: 0
|
||||
fields:
|
||||
- field: hasLeader
|
||||
type: 0
|
||||
i18n:
|
||||
zh-CN: 是否存在领导者(1有0无)
|
||||
en-US: Has Leader(1=yes 0=no)
|
||||
ja-JP: リーダー有無(1=有 0=無)
|
||||
aliasFields:
|
||||
- value
|
||||
calculates:
|
||||
- hasLeader=value
|
||||
protocol: http
|
||||
http:
|
||||
host: ^_^host^_^
|
||||
port: ^_^port^_^
|
||||
url: /metrics
|
||||
timeout: ^_^timeout^_^
|
||||
ssl: ^_^ssl^_^
|
||||
method: GET
|
||||
headers:
|
||||
^_^headers^_^: ^_^headers^_^
|
||||
authorization:
|
||||
type: ^_^authType^_^
|
||||
basicAuthUsername: ^_^username^_^
|
||||
basicAuthPassword: ^_^password^_^
|
||||
digestAuthUsername: ^_^username^_^
|
||||
digestAuthPassword: ^_^password^_^
|
||||
parseType: prometheus
|
||||
# metrics - etcd_mvcc_db_total_size_in_bytes
|
||||
- name: etcd_mvcc_db_total_size_in_bytes
|
||||
i18n:
|
||||
zh-CN: 数据库大小
|
||||
en-US: Database Size
|
||||
ja-JP: データベースサイズ
|
||||
priority: 1
|
||||
fields:
|
||||
- field: dbSize
|
||||
type: 0
|
||||
unit: MB
|
||||
i18n:
|
||||
zh-CN: 物理分配的数据库大小
|
||||
en-US: Physically Allocated DB Size
|
||||
ja-JP: 物理割当済みDBサイズ
|
||||
aliasFields:
|
||||
- value
|
||||
calculates:
|
||||
- dbSize=value
|
||||
units:
|
||||
- dbSize=B->MB
|
||||
protocol: http
|
||||
http:
|
||||
host: ^_^host^_^
|
||||
port: ^_^port^_^
|
||||
url: /metrics
|
||||
timeout: ^_^timeout^_^
|
||||
ssl: ^_^ssl^_^
|
||||
method: GET
|
||||
headers:
|
||||
^_^headers^_^: ^_^headers^_^
|
||||
authorization:
|
||||
type: ^_^authType^_^
|
||||
basicAuthUsername: ^_^username^_^
|
||||
basicAuthPassword: ^_^password^_^
|
||||
digestAuthUsername: ^_^username^_^
|
||||
digestAuthPassword: ^_^password^_^
|
||||
parseType: prometheus
|
||||
# metrics - etcd_server_leader_changes_seen_total
|
||||
- name: etcd_server_leader_changes_seen_total
|
||||
i18n:
|
||||
zh-CN: 领导者变更次数
|
||||
en-US: Leader Changes
|
||||
ja-JP: リーダー変更回数
|
||||
priority: 1
|
||||
fields:
|
||||
- field: leaderChanges
|
||||
type: 0
|
||||
i18n:
|
||||
zh-CN: 已观测到的领导者变更总次数
|
||||
en-US: Total Leader Changes Seen
|
||||
ja-JP: 観測されたリーダー変更総数
|
||||
aliasFields:
|
||||
- value
|
||||
calculates:
|
||||
- leaderChanges=value
|
||||
protocol: http
|
||||
http:
|
||||
host: ^_^host^_^
|
||||
port: ^_^port^_^
|
||||
url: /metrics
|
||||
timeout: ^_^timeout^_^
|
||||
ssl: ^_^ssl^_^
|
||||
method: GET
|
||||
headers:
|
||||
^_^headers^_^: ^_^headers^_^
|
||||
authorization:
|
||||
type: ^_^authType^_^
|
||||
basicAuthUsername: ^_^username^_^
|
||||
basicAuthPassword: ^_^password^_^
|
||||
digestAuthUsername: ^_^username^_^
|
||||
digestAuthPassword: ^_^password^_^
|
||||
parseType: prometheus
|
||||
# metrics - process_cpu_seconds_total (standard Go/Prometheus client process collector, always present)
|
||||
- name: process_cpu_seconds_total
|
||||
i18n:
|
||||
zh-CN: 进程CPU时间
|
||||
en-US: Process CPU Time
|
||||
ja-JP: プロセスCPU時間
|
||||
priority: 1
|
||||
fields:
|
||||
- field: cpuSeconds
|
||||
type: 0
|
||||
unit: s
|
||||
i18n:
|
||||
zh-CN: 累计用户与系统CPU时间
|
||||
en-US: Total User+System CPU Time
|
||||
ja-JP: 累計ユーザー+システムCPU時間
|
||||
aliasFields:
|
||||
- value
|
||||
calculates:
|
||||
- cpuSeconds=value
|
||||
protocol: http
|
||||
http:
|
||||
host: ^_^host^_^
|
||||
port: ^_^port^_^
|
||||
url: /metrics
|
||||
timeout: ^_^timeout^_^
|
||||
ssl: ^_^ssl^_^
|
||||
method: GET
|
||||
headers:
|
||||
^_^headers^_^: ^_^headers^_^
|
||||
authorization:
|
||||
type: ^_^authType^_^
|
||||
basicAuthUsername: ^_^username^_^
|
||||
basicAuthPassword: ^_^password^_^
|
||||
digestAuthUsername: ^_^username^_^
|
||||
digestAuthPassword: ^_^password^_^
|
||||
parseType: prometheus
|
||||
# metrics - process_resident_memory_bytes
|
||||
- name: process_resident_memory_bytes
|
||||
i18n:
|
||||
zh-CN: 进程内存占用
|
||||
en-US: Process Resident Memory
|
||||
ja-JP: プロセス常駐メモリ
|
||||
priority: 1
|
||||
fields:
|
||||
- field: memory
|
||||
type: 0
|
||||
unit: MB
|
||||
i18n:
|
||||
zh-CN: 常驻内存大小
|
||||
en-US: Resident Memory Size
|
||||
ja-JP: 常駐メモリサイズ
|
||||
aliasFields:
|
||||
- value
|
||||
calculates:
|
||||
- memory=value
|
||||
units:
|
||||
- memory=B->MB
|
||||
protocol: http
|
||||
http:
|
||||
host: ^_^host^_^
|
||||
port: ^_^port^_^
|
||||
url: /metrics
|
||||
timeout: ^_^timeout^_^
|
||||
ssl: ^_^ssl^_^
|
||||
method: GET
|
||||
headers:
|
||||
^_^headers^_^: ^_^headers^_^
|
||||
authorization:
|
||||
type: ^_^authType^_^
|
||||
basicAuthUsername: ^_^username^_^
|
||||
basicAuthPassword: ^_^password^_^
|
||||
digestAuthUsername: ^_^username^_^
|
||||
digestAuthPassword: ^_^password^_^
|
||||
parseType: prometheus
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.config;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Test case for {@link AuthorizedSwaggerIndexTransformer}.
|
||||
*/
|
||||
class AuthorizedSwaggerIndexTransformerTest {
|
||||
|
||||
@Test
|
||||
void shouldAttachTheHertzBeatTokenToSameOriginRequests() {
|
||||
final String initializer = """
|
||||
window.ui = SwaggerUIBundle({
|
||||
configUrl: "/v3/api-docs/swagger-config",
|
||||
presets: [SwaggerUIBundle.presets.apis]
|
||||
});
|
||||
""";
|
||||
|
||||
final String transformed = AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer);
|
||||
|
||||
assertTrue(transformed.contains("window.localStorage.getItem('Authorization')"));
|
||||
assertTrue(transformed.contains("request.headers['Authorization'] = `Bearer ${token}`"));
|
||||
assertTrue(transformed.contains("sameOrigin && token"));
|
||||
assertTrue(transformed.contains("presets: [SwaggerUIBundle.presets.apis]"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailClosedWhenTheSwaggerInitializerShapeChanges() {
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor("window.ui = {};"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Springdoc writes an interceptor of its own when csrf support is turned on, and it
|
||||
* can be configured to write to the {@code Authorization} header too. The last write
|
||||
* wins, so ours has to come after the one already there.
|
||||
*/
|
||||
@Test
|
||||
void shouldComposeWithAnExistingRequestInterceptor() {
|
||||
final String initializer = """
|
||||
window.ui = SwaggerUIBundle({
|
||||
requestInterceptor: (request) => {
|
||||
request.headers['Authorization'] = 'csrf';
|
||||
return request;
|
||||
},
|
||||
presets: [SwaggerUIBundle.presets.apis]
|
||||
});
|
||||
""";
|
||||
|
||||
final String transformed = AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer);
|
||||
|
||||
assertTrue(transformed.contains("request.headers['Authorization'] = 'csrf'"));
|
||||
assertTrue(transformed.indexOf("request.headers['Authorization'] = `Bearer ${token}`")
|
||||
> transformed.indexOf("request.headers['Authorization'] = 'csrf'"),
|
||||
"the token has to be written after the interceptor that was already there");
|
||||
assertEquals(1, countOf(transformed, "requestInterceptor:"),
|
||||
"a second key of the same name would drop one of the two interceptors");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldFailClosedWhenTheExistingInterceptorDoesNotReturnTheRequest() {
|
||||
final String initializer = """
|
||||
window.ui = SwaggerUIBundle({
|
||||
requestInterceptor: (request) => {
|
||||
presets: [SwaggerUIBundle.presets.apis]
|
||||
""";
|
||||
|
||||
assertThrows(IllegalStateException.class,
|
||||
() -> AuthorizedSwaggerIndexTransformer.addAuthorizationInterceptor(initializer));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param initializer the transformed script
|
||||
* @param token the substring to count
|
||||
* @return how many times the substring occurs
|
||||
*/
|
||||
private static int countOf(String initializer, String token) {
|
||||
int count = 0;
|
||||
for (int index = initializer.indexOf(token); index >= 0; index = initializer.indexOf(token, index + 1)) {
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.config;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import jakarta.servlet.Filter;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.web.servlet.FilterRegistrationBean;
|
||||
import org.springframework.mock.web.MockHttpServletRequest;
|
||||
import org.springframework.mock.web.MockHttpServletResponse;
|
||||
import org.springframework.mock.web.MockFilterChain;
|
||||
|
||||
/**
|
||||
* Test case for {@link SecurityCorsConfiguration}.
|
||||
*
|
||||
* <p>The filter answers every origin, which is intentional, and requests authenticate with
|
||||
* a token in the Authorization header rather than a cookie, so credentials do not need to
|
||||
* be allowed. Both halves are asserted: the credentials header is not sent, and a preflight
|
||||
* still succeeds so the api stays reachable cross origin.
|
||||
*/
|
||||
class SecurityCorsConfigurationTest {
|
||||
|
||||
private static final String OTHER_ORIGIN = "https://other.example";
|
||||
|
||||
@Test
|
||||
void testCredentialsAreNotAllowedForCrossOriginRequests() throws Exception {
|
||||
MockHttpServletResponse response = handlePreflight();
|
||||
|
||||
assertNotEquals("true", response.getHeader("Access-Control-Allow-Credentials"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCrossOriginRequestsAreStillAnswered() throws Exception {
|
||||
MockHttpServletResponse response = handlePreflight();
|
||||
|
||||
assertNotNull(response.getHeader("Access-Control-Allow-Origin"),
|
||||
"the api is meant to stay reachable cross origin");
|
||||
assertEquals(200, response.getStatus());
|
||||
}
|
||||
|
||||
private MockHttpServletResponse handlePreflight() throws Exception {
|
||||
FilterRegistrationBean<?> registration = new SecurityCorsConfiguration().corsFilter();
|
||||
Filter filter = (Filter) registration.getFilter();
|
||||
|
||||
MockHttpServletRequest request = new MockHttpServletRequest("OPTIONS", "/api/monitors");
|
||||
request.addHeader("Origin", OTHER_ORIGIN);
|
||||
request.addHeader("Access-Control-Request-Method", "GET");
|
||||
MockHttpServletResponse response = new MockHttpServletResponse();
|
||||
|
||||
filter.doFilter(request, response, new MockFilterChain());
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -86,9 +86,16 @@ resourceRole:
|
||||
- /api/ingestion/otlp/**===get===[admin,user,guest]
|
||||
- /api/logs/**===get===[admin,user,guest]
|
||||
- /api/traces/**===get===[admin,user,guest]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -121,10 +128,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
+289
-26
@@ -19,13 +19,19 @@
|
||||
|
||||
package org.apache.hertzbeat.push.service.impl;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import jakarta.annotation.Nullable;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.prometheus.parser.MetricFamily;
|
||||
@@ -37,6 +43,7 @@ import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
|
||||
import org.apache.hertzbeat.push.dao.PushMonitorDao;
|
||||
import org.apache.hertzbeat.push.service.PushGatewayService;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
@@ -46,26 +53,81 @@ import org.springframework.stereotype.Service;
|
||||
@Slf4j
|
||||
@Service
|
||||
public class PushGatewayServiceImpl implements PushGatewayService {
|
||||
|
||||
|
||||
private static final byte PUSH_MONITOR_TYPE = (byte) 1;
|
||||
|
||||
private final CommonDataQueue commonDataQueue;
|
||||
|
||||
private final PushMonitorDao pushMonitorDao;
|
||||
|
||||
private final Map<String, Long> jobInstanceMap;
|
||||
|
||||
public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao) {
|
||||
private final Map<JobInstance, Long> jobInstanceMap;
|
||||
|
||||
/**
|
||||
* Cap on push monitors created automatically from unknown job/instance pairs.
|
||||
*
|
||||
* <p>The route is unauthenticated by design, and every new pair used to persist a
|
||||
* monitor row and add a `jobInstanceMap` entry that is never removed, so a caller
|
||||
* iterating over made up names could grow the database and the heap without bound.
|
||||
* Above the cap an unknown pair is refused while the pairs already known keep working,
|
||||
* which is why eviction is not used here: evicting a live entry would make the next
|
||||
* push for that pair create a second monitor for the same job and instance.
|
||||
*/
|
||||
private final int maxAutoCreatedMonitors;
|
||||
|
||||
/**
|
||||
* Cap on how many bytes a single push body may carry.
|
||||
*
|
||||
* <p>The parser materializes its result in memory, and the servlet container does not bound
|
||||
* a non form request body, so an independent byte limit is still needed alongside the sample
|
||||
* limit to keep long names and label values from exhausting the heap.
|
||||
*/
|
||||
private final long maxBodyBytes;
|
||||
|
||||
/**
|
||||
* Cap on how many samples a single push body may carry. The parser stops before allocating
|
||||
* a sample beyond this limit, so a compact body cannot create an unbounded object graph.
|
||||
*/
|
||||
private final int maxSamples;
|
||||
|
||||
/**
|
||||
* One entry per pair whose monitor is being created, so that concurrent pushes naming the
|
||||
* same unknown pair wait for one creation instead of each starting their own. Persistence
|
||||
* stays outside any shared lock: a slow database would otherwise hold every request for an
|
||||
* unknown pair, and each of those requests is already holding its parsed samples.
|
||||
*/
|
||||
private final Map<JobInstance, CompletableFuture<Long>> monitorCreationMap;
|
||||
|
||||
/**
|
||||
* Successful and in flight creations together, so that the cap is claimed before the
|
||||
* database write rather than counted after it.
|
||||
*/
|
||||
private final AtomicInteger trackedMonitorCount;
|
||||
|
||||
public PushGatewayServiceImpl(CommonDataQueue commonDataQueue, PushMonitorDao pushMonitorDao,
|
||||
@Value("${hertzbeat.push.max-auto-created-monitors:10000}") int maxAutoCreatedMonitors,
|
||||
@Value("${hertzbeat.push.max-body-bytes:5242880}") long maxBodyBytes,
|
||||
@Value("${hertzbeat.push.max-samples:10000}") int maxSamples) {
|
||||
if (maxAutoCreatedMonitors < 0 || maxBodyBytes < 0 || maxSamples < 0) {
|
||||
throw new IllegalArgumentException("push gateway limits must not be negative");
|
||||
}
|
||||
this.commonDataQueue = commonDataQueue;
|
||||
this.pushMonitorDao = pushMonitorDao;
|
||||
this.maxAutoCreatedMonitors = maxAutoCreatedMonitors;
|
||||
this.maxBodyBytes = maxBodyBytes;
|
||||
this.maxSamples = maxSamples;
|
||||
jobInstanceMap = new ConcurrentHashMap<>();
|
||||
pushMonitorDao.findMonitorsByType((byte) 1).forEach(monitor ->
|
||||
jobInstanceMap.put(monitor.getApp() + "_" + monitor.getName(), monitor.getId()));
|
||||
pushMonitorDao.findMonitorsByType(PUSH_MONITOR_TYPE).forEach(monitor ->
|
||||
jobInstanceMap.put(new JobInstance(monitor.getApp(), monitor.getName()), monitor.getId()));
|
||||
monitorCreationMap = new ConcurrentHashMap<>();
|
||||
trackedMonitorCount = new AtomicInteger(jobInstanceMap.size());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean pushPrometheusMetrics(InputStream inputStream, String job, String instance) {
|
||||
try {
|
||||
long curTime = Instant.now().toEpochMilli();
|
||||
Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(inputStream);
|
||||
final long curTime = Instant.now().toEpochMilli();
|
||||
final Map<String, MetricFamily> metricFamilyMap = OnlineParser.parseMetrics(
|
||||
new BoundedInputStream(inputStream, maxBodyBytes), maxSamples);
|
||||
if (metricFamilyMap == null) {
|
||||
log.error("parse prometheus metrics is null, job: {}, instance: {}", job, instance);
|
||||
return false;
|
||||
@@ -74,20 +136,11 @@ public class PushGatewayServiceImpl implements PushGatewayService {
|
||||
if (job != null && instance != null) {
|
||||
// auto create monitor when job and instance not null
|
||||
// job is app, instance is the name
|
||||
id = jobInstanceMap.computeIfAbsent(job + "_" + instance, key -> {
|
||||
log.info("auto create monitor by prometheus push, job: {}, instance: {}", job, instance);
|
||||
long monitorId = SnowFlakeIdGenerator.generateId();
|
||||
Monitor monitor = Monitor.builder()
|
||||
.id(monitorId)
|
||||
.app(job)
|
||||
.name(instance)
|
||||
.instance(instance)
|
||||
.type((byte) 1)
|
||||
.status(CommonConstants.MONITOR_UP_CODE)
|
||||
.build();
|
||||
this.pushMonitorDao.save(monitor);
|
||||
return monitorId;
|
||||
});
|
||||
final Long monitorId = resolveMonitorId(new JobInstance(job, instance));
|
||||
if (monitorId == null) {
|
||||
return false;
|
||||
}
|
||||
id = monitorId;
|
||||
}
|
||||
for (Map.Entry<String, MetricFamily> entry : metricFamilyMap.entrySet()) {
|
||||
CollectRep.MetricsData.Builder builder = CollectRep.MetricsData.newBuilder();
|
||||
@@ -110,9 +163,18 @@ public class PushGatewayServiceImpl implements PushGatewayService {
|
||||
builder.addField(CollectRep.Field.newBuilder().setName("value")
|
||||
.setType(CommonConstants.TYPE_NUMBER).setLabel(false).build());
|
||||
}
|
||||
Map<String, String> labelMap = metric.getLabels()
|
||||
.stream()
|
||||
.collect(Collectors.toMap(MetricFamily.Label::getName, MetricFamily.Label::getValue));
|
||||
// A repeated label name is refused rather than resolved: the exposition
|
||||
// format requires the names of a label set to be unique, and keeping one
|
||||
// of the values would emit a schema carrying that name twice. Built by
|
||||
// hand so the refusal is a rejection this method can answer with a
|
||||
// warning, not the error trace a collector's exception would produce.
|
||||
Map<String, String> labelMap = new HashMap<>(metric.getLabels().size());
|
||||
for (MetricFamily.Label label : metric.getLabels()) {
|
||||
if (labelMap.containsKey(label.getName())) {
|
||||
throw new DuplicateLabelException(label.getName());
|
||||
}
|
||||
labelMap.put(label.getName(), label.getValue());
|
||||
}
|
||||
CollectRep.ValueRow.Builder valueRowBuilder = CollectRep.ValueRow.newBuilder();
|
||||
for (String field : metricsFields) {
|
||||
String fieldValue = labelMap.get(field);
|
||||
@@ -125,9 +187,210 @@ public class PushGatewayServiceImpl implements PushGatewayService {
|
||||
}
|
||||
}
|
||||
return true;
|
||||
} catch (BodyTooLargeException e) {
|
||||
// A rejection, not a failure: caught apart from the generic handler below so that a
|
||||
// caller repeating oversized bodies costs one warning line each, not a stack trace
|
||||
log.warn("reject prometheus push over the {} byte body limit, job: {}, instance: {}",
|
||||
maxBodyBytes, job, instance);
|
||||
return false;
|
||||
} catch (OnlineParser.SampleLimitExceededException e) {
|
||||
log.warn("reject prometheus push over the {} sample limit, job: {}, instance: {}",
|
||||
maxSamples, job, instance);
|
||||
return false;
|
||||
} catch (DuplicateLabelException e) {
|
||||
log.warn("reject prometheus push repeating a label name, job: {}, instance: {}: {}",
|
||||
job, instance, e.getMessage());
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
log.error("push prometheus metrics error", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the monitor id a job/instance pair resolves to, creating the monitor on first
|
||||
* sight, or null once {@link #maxAutoCreatedMonitors} is reached.
|
||||
*
|
||||
* <p>Concurrent pushes naming the same unknown pair share one creation through a future, and
|
||||
* the cap is claimed by an atomic count before the database write. Nothing here holds a lock
|
||||
* across that write: unrelated pairs persist in parallel, and a slow database delays only the
|
||||
* requests naming the pair being created, which matters because every waiting request is
|
||||
* holding the samples it already parsed.
|
||||
*
|
||||
* @param pair Job and instance the push named
|
||||
* @return The monitor id, or null when the cap leaves no room for a new one
|
||||
*/
|
||||
@Nullable
|
||||
private Long resolveMonitorId(JobInstance pair) {
|
||||
final Long known = jobInstanceMap.get(pair);
|
||||
if (known != null) {
|
||||
return known;
|
||||
}
|
||||
|
||||
final CompletableFuture<Long> proposedCreation = new CompletableFuture<>();
|
||||
final CompletableFuture<Long> ongoingCreation = monitorCreationMap.putIfAbsent(pair, proposedCreation);
|
||||
if (ongoingCreation != null) {
|
||||
try {
|
||||
return ongoingCreation.join();
|
||||
} catch (CompletionException e) {
|
||||
// The request owning the creation reports the failure once; joining its exception
|
||||
// here would multiply a single database error by every request that waited
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Looked up again now that the creation is claimed: another request may have finished
|
||||
// this pair between the lookup above and this claim, and going on to create it would
|
||||
// leave two monitors for one pair and spend a second slot of the cap
|
||||
final Long createdMeanwhile = jobInstanceMap.get(pair);
|
||||
if (createdMeanwhile != null) {
|
||||
proposedCreation.complete(createdMeanwhile);
|
||||
return createdMeanwhile;
|
||||
}
|
||||
if (!reserveMonitorSlot()) {
|
||||
proposedCreation.complete(null);
|
||||
log.warn("reject prometheus push for unknown job: {}, instance: {}, "
|
||||
+ "already tracking {} push monitors, limit is {}",
|
||||
pair.job(), pair.instance(), trackedMonitorCount.get(), maxAutoCreatedMonitors);
|
||||
return null;
|
||||
}
|
||||
boolean created = false;
|
||||
try {
|
||||
final long monitorId = createMonitor(pair);
|
||||
jobInstanceMap.put(pair, monitorId);
|
||||
created = true;
|
||||
proposedCreation.complete(monitorId);
|
||||
return monitorId;
|
||||
} finally {
|
||||
if (!created) {
|
||||
trackedMonitorCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
} catch (RuntimeException | Error e) {
|
||||
proposedCreation.completeExceptionally(e);
|
||||
throw e;
|
||||
} finally {
|
||||
monitorCreationMap.remove(pair, proposedCreation);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Claims one slot of the cap, or reports that none is left. Claiming before the database
|
||||
* write is what keeps concurrent creations from exceeding the cap together.
|
||||
*/
|
||||
private boolean reserveMonitorSlot() {
|
||||
int tracked = trackedMonitorCount.get();
|
||||
while (tracked < maxAutoCreatedMonitors) {
|
||||
if (trackedMonitorCount.compareAndSet(tracked, tracked + 1)) {
|
||||
return true;
|
||||
}
|
||||
tracked = trackedMonitorCount.get();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persists a push monitor after its slot of the cap has been claimed.
|
||||
*/
|
||||
private long createMonitor(JobInstance pair) {
|
||||
final String job = pair.job();
|
||||
final String instance = pair.instance();
|
||||
log.info("auto create monitor by prometheus push, job: {}, instance: {}", job, instance);
|
||||
final long monitorId = SnowFlakeIdGenerator.generateId();
|
||||
final Monitor monitor = Monitor.builder()
|
||||
.id(monitorId)
|
||||
.app(job)
|
||||
.name(instance)
|
||||
.instance(instance)
|
||||
.type(PUSH_MONITOR_TYPE)
|
||||
.status(CommonConstants.MONITOR_UP_CODE)
|
||||
.build();
|
||||
this.pushMonitorDao.save(monitor);
|
||||
return monitorId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Identifies the monitor a push belongs to.
|
||||
*
|
||||
* <p>The two names are kept apart instead of being joined into one string: a separator
|
||||
* carries no meaning in either name, so `job + "_" + instance` maps ("a", "b_c") and
|
||||
* ("a_b", "c") onto the same key. Colliding pairs would push their samples into whichever
|
||||
* monitor was created first, and at startup they would collapse into a single map entry,
|
||||
* making the cap count fewer monitors than the database actually holds.
|
||||
*/
|
||||
private record JobInstance(String job, String instance) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Raised when a sample repeats a label name, which the exposition format does not allow.
|
||||
* Kept apart from the generic handler so a malformed body costs one warning line rather
|
||||
* than an error trace on a route that takes its input from anyone.
|
||||
*/
|
||||
static final class DuplicateLabelException extends IOException {
|
||||
|
||||
DuplicateLabelException(String name) {
|
||||
super("sample repeats the label name " + name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Raised when a body goes past {@link #maxBodyBytes}. It is kept apart from the other read
|
||||
* failures so the caller can answer a body that is merely too large without an error trace.
|
||||
*/
|
||||
static final class BodyTooLargeException extends IOException {
|
||||
|
||||
BodyTooLargeException(long limit) {
|
||||
super("push body exceeds the " + limit + " byte limit");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fails the read once the body has delivered more than {@code limit} bytes, instead of
|
||||
* letting the parser accumulate an unbounded body in memory. Reading stops at the
|
||||
* failure, so the bytes beyond the limit are never buffered.
|
||||
*/
|
||||
static final class BoundedInputStream extends InputStream {
|
||||
|
||||
private final InputStream delegate;
|
||||
|
||||
private final long limit;
|
||||
|
||||
private long bytesRead;
|
||||
|
||||
BoundedInputStream(InputStream delegate, long limit) {
|
||||
this.delegate = delegate;
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read() throws IOException {
|
||||
final int value = delegate.read();
|
||||
if (value != -1) {
|
||||
recordBytesRead(1);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] buffer, int offset, int length) throws IOException {
|
||||
final int bytesReadNow = delegate.read(buffer, offset, length);
|
||||
if (bytesReadNow > 0) {
|
||||
recordBytesRead(bytesReadNow);
|
||||
}
|
||||
return bytesReadNow;
|
||||
}
|
||||
|
||||
private void recordBytesRead(int increment) throws IOException {
|
||||
bytesRead += increment;
|
||||
if (bytesRead > limit) {
|
||||
throw new BodyTooLargeException(limit);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
delegate.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+367
@@ -0,0 +1,367 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.push.service.impl;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CyclicBarrier;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.apache.hertzbeat.common.entity.manager.Monitor;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.push.dao.PushMonitorDao;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
/**
|
||||
* Test case for {@link PushGatewayServiceImpl}.
|
||||
*
|
||||
* <p>`/api/push/prometheus/**` is unauthenticated by design, so the resource a single
|
||||
* anonymous request may consume has to be bounded: the body it may carry, the samples it
|
||||
* may enqueue, and the number of push monitors it may bring into existence.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class PushGatewayServiceImplTest {
|
||||
|
||||
private static final String BODY = "sample_metric{label=\"a\"} 1\n";
|
||||
|
||||
@Mock
|
||||
private CommonDataQueue commonDataQueue;
|
||||
|
||||
@Mock
|
||||
private PushMonitorDao pushMonitorDao;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
// The stream test below builds no service, so this default must not be strict
|
||||
lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of());
|
||||
}
|
||||
|
||||
private PushGatewayServiceImpl createService(int maxMonitors, long maxBodyBytes, int maxSamples) {
|
||||
return new PushGatewayServiceImpl(commonDataQueue, pushMonitorDao, maxMonitors, maxBodyBytes, maxSamples);
|
||||
}
|
||||
|
||||
private static ByteArrayInputStream createBody(String content) {
|
||||
return new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
/**
|
||||
* The exposition format requires the names of a label set to be unique, so a sample that
|
||||
* repeats one is refused rather than resolved. It stays a rejection though: answering it
|
||||
* with an error trace would let a malformed body fill the log on an anonymous route.
|
||||
*/
|
||||
@Test
|
||||
void testSampleRepeatingTheLabelNameIsRejected() {
|
||||
final PushGatewayServiceImpl service = createService(10, 1024, 100);
|
||||
|
||||
assertFalse(service.pushPrometheusMetrics(
|
||||
createBody("sample_metric{label=\"a\",label=\"b\"} 1\n"), "job1", "instance1"));
|
||||
|
||||
verify(commonDataQueue, never()).sendMetricsData(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPushIsAcceptedWithinTheLimits() {
|
||||
final PushGatewayServiceImpl service = createService(10, 1024, 100);
|
||||
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
|
||||
verify(pushMonitorDao).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBodyBeyondTheByteLimitIsRejected() {
|
||||
final PushGatewayServiceImpl service = createService(10, 16, 100);
|
||||
|
||||
assertFalse(service.pushPrometheusMetrics(createBody(BODY.repeat(100)), "job1", "instance1"));
|
||||
|
||||
verify(pushMonitorDao, never()).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBodyBeyondTheSampleLimitIsRejected() {
|
||||
final PushGatewayServiceImpl service = createService(10, 1024 * 1024, 2);
|
||||
final StringBuilder many = new StringBuilder();
|
||||
for (int index = 0; index < 10; index++) {
|
||||
many.append("sample_metric{label=\"value").append(index).append("\"} 1\n");
|
||||
}
|
||||
final ByteArrayInputStream inputStream = createBody(many.toString());
|
||||
|
||||
assertFalse(service.pushPrometheusMetrics(inputStream, "job1", "instance1"));
|
||||
assertTrue(inputStream.available() > 0, "the parser should stop before consuming the remaining samples");
|
||||
|
||||
verify(pushMonitorDao, never()).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBodyAtTheSampleLimitIsAccepted() {
|
||||
final PushGatewayServiceImpl service = createService(10, 1024, 2);
|
||||
final String twoSamples = "sample_metric{label=\"a\"} 1\n"
|
||||
+ "sample_metric{label=\"b\"} 2\n";
|
||||
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(twoSamples), "job1", "instance1"));
|
||||
}
|
||||
|
||||
/**
|
||||
* An unknown job/instance pair persists a monitor row and adds a map entry that is
|
||||
* never removed, so without a cap an anonymous caller iterating over made up names
|
||||
* grows the database and the heap without bound.
|
||||
*/
|
||||
@Test
|
||||
void testAutoCreationStopsAtTheMonitorLimit() {
|
||||
final PushGatewayServiceImpl service = createService(2, 1024, 100);
|
||||
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
|
||||
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job3", "instance3"));
|
||||
|
||||
verify(pushMonitorDao, times(2)).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* The cap must not turn into eviction: a pair already known has to keep resolving to
|
||||
* the monitor it created, otherwise a later push would create a second monitor for the
|
||||
* same job and instance.
|
||||
*/
|
||||
@Test
|
||||
void testKnownPairsKeepWorkingAtTheLimit() {
|
||||
final PushGatewayServiceImpl service = createService(1, 1024, 100);
|
||||
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "other", "instance"));
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
|
||||
verify(pushMonitorDao, times(1)).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* The route is anonymous, so nothing stops a caller from sending its unknown pairs all at
|
||||
* once. Testing the cap and claiming the entry in two steps lets every request that already
|
||||
* passed the test create a monitor of its own, which is the cap being exceeded by as many
|
||||
* requests as the container serves in parallel.
|
||||
*/
|
||||
@Test
|
||||
void testConcurrentPushesForUnknownPairsStopAtTheMonitorLimit() throws Exception {
|
||||
final int callers = 16;
|
||||
final PushGatewayServiceImpl service = createService(1, 1024, 100);
|
||||
final CyclicBarrier startTogether = new CyclicBarrier(callers);
|
||||
final ExecutorService pool = Executors.newFixedThreadPool(callers);
|
||||
final AtomicInteger accepted = new AtomicInteger();
|
||||
try {
|
||||
final List<Future<?>> pushes = new ArrayList<>();
|
||||
for (int index = 0; index < callers; index++) {
|
||||
final String instance = "instance" + index;
|
||||
pushes.add(pool.submit(() -> {
|
||||
startTogether.await();
|
||||
if (service.pushPrometheusMetrics(createBody(BODY), "job", instance)) {
|
||||
accepted.incrementAndGet();
|
||||
}
|
||||
return null;
|
||||
}));
|
||||
}
|
||||
for (final Future<?> push : pushes) {
|
||||
push.get(30, TimeUnit.SECONDS);
|
||||
}
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
assertEquals(1, accepted.get());
|
||||
verify(pushMonitorDao, times(1)).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFailedSaveDoesNotConsumeTheMonitorLimit() {
|
||||
when(pushMonitorDao.save(any(Monitor.class)))
|
||||
.thenThrow(new IllegalStateException("database unavailable"))
|
||||
.thenAnswer(invocation -> invocation.getArgument(0));
|
||||
final PushGatewayServiceImpl service = createService(1, 1024, 100);
|
||||
|
||||
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
|
||||
|
||||
verify(pushMonitorDao, times(2)).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testNegativeLimitsAreRejectedAtConstruction() {
|
||||
assertThrows(IllegalArgumentException.class, () -> createService(-1, 1024, 100));
|
||||
assertThrows(IllegalArgumentException.class, () -> createService(1, -1, 100));
|
||||
assertThrows(IllegalArgumentException.class, () -> createService(1, 1024, -1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testConcurrentPushesForTheSamePairCreateOneMonitor() throws Exception {
|
||||
final int callers = 8;
|
||||
final CyclicBarrier startTogether = new CyclicBarrier(callers);
|
||||
final PushGatewayServiceImpl service = createService(1, 1024, 100);
|
||||
final ExecutorService pool = Executors.newFixedThreadPool(callers);
|
||||
try {
|
||||
final List<Future<Boolean>> pushes = new ArrayList<>();
|
||||
for (int index = 0; index < callers; index++) {
|
||||
pushes.add(pool.submit(() -> {
|
||||
startTogether.await();
|
||||
return service.pushPrometheusMetrics(createBody(BODY), "job", "instance");
|
||||
}));
|
||||
}
|
||||
for (final Future<Boolean> push : pushes) {
|
||||
assertTrue(push.get(30, TimeUnit.SECONDS));
|
||||
}
|
||||
} finally {
|
||||
pool.shutdownNow();
|
||||
}
|
||||
|
||||
verify(pushMonitorDao).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* A request may read no entry for a pair and only then claim the creation, by which time
|
||||
* another request may have created that pair and cleared its claim. Without a second lookup
|
||||
* once the claim is won, this request goes on to create the same pair again, leaving two
|
||||
* monitors for one pair and a slot of the cap spent for good.
|
||||
*
|
||||
* <p>The interleaving is forced rather than raced: the map hands back a miss, and while it
|
||||
* does, a competing push runs its whole creation.
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
void testStaleMissDoesNotCreateTwoMonitorsForOnePair() throws Exception {
|
||||
final PushGatewayServiceImpl service = createService(2, 1024, 100);
|
||||
final Field trackedPairs = PushGatewayServiceImpl.class.getDeclaredField("jobInstanceMap");
|
||||
trackedPairs.setAccessible(true);
|
||||
final AtomicBoolean competed = new AtomicBoolean();
|
||||
final Map probing = new ConcurrentHashMap() {
|
||||
@Override
|
||||
public Object get(Object key) {
|
||||
final Object value = super.get(key);
|
||||
if (value == null && competed.compareAndSet(false, true)) {
|
||||
service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
};
|
||||
trackedPairs.set(service, probing);
|
||||
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
|
||||
verify(pushMonitorDao, times(1)).save(any(Monitor.class));
|
||||
assertEquals(1, probing.size());
|
||||
// The slot the duplicate would have taken is still there for another pair
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMonitorsLoadedAtStartupCountTowardsTheLimit() {
|
||||
lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of(
|
||||
Monitor.builder().id(1L).app("job1").name("instance1").build()));
|
||||
final PushGatewayServiceImpl service = createService(1, 1024, 100);
|
||||
|
||||
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job2", "instance2"));
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job1", "instance1"));
|
||||
|
||||
verify(pushMonitorDao, never()).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
/**
|
||||
* A separator carries no meaning inside a job or an instance name, so the two names must not
|
||||
* be joined into a single key: ("job", "a_b") and ("job_a", "b") are different monitors, and
|
||||
* the second pair must not push its samples into the monitor the first one created.
|
||||
*/
|
||||
@Test
|
||||
void testPairsSharingTheSeparatorAreDistinctMonitors() {
|
||||
final PushGatewayServiceImpl service = createService(10, 1024, 100);
|
||||
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job", "a_b"));
|
||||
assertTrue(service.pushPrometheusMetrics(createBody(BODY), "job_a", "b"));
|
||||
|
||||
verify(pushMonitorDao, times(2)).save(any(Monitor.class));
|
||||
final ArgumentCaptor<CollectRep.MetricsData> pushed =
|
||||
ArgumentCaptor.forClass(CollectRep.MetricsData.class);
|
||||
verify(commonDataQueue, times(2)).sendMetricsData(pushed.capture());
|
||||
assertNotEquals(pushed.getAllValues().get(0).getId(), pushed.getAllValues().get(1).getId());
|
||||
}
|
||||
|
||||
/**
|
||||
* Colliding pairs must not collapse into one entry while the monitors are loaded either,
|
||||
* which would let the cap count fewer monitors than the database actually holds.
|
||||
*/
|
||||
@Test
|
||||
void testPairsSharingTheSeparatorCountSeparatelyAtStartup() {
|
||||
lenient().when(pushMonitorDao.findMonitorsByType((byte) 1)).thenReturn(List.of(
|
||||
Monitor.builder().id(1L).app("job").name("a_b").build(),
|
||||
Monitor.builder().id(2L).app("job_a").name("b").build()));
|
||||
final PushGatewayServiceImpl service = createService(2, 1024, 100);
|
||||
|
||||
assertFalse(service.pushPrometheusMetrics(createBody(BODY), "job3", "instance3"));
|
||||
|
||||
verify(pushMonitorDao, never()).save(any(Monitor.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBoundedStreamStopsAtTheLimit() throws Exception {
|
||||
final PushGatewayServiceImpl.BoundedInputStream stream =
|
||||
new PushGatewayServiceImpl.BoundedInputStream(createBody("abcdef"), 3);
|
||||
|
||||
assertEquals('a', stream.read());
|
||||
assertEquals('b', stream.read());
|
||||
assertEquals('c', stream.read());
|
||||
assertThrows(PushGatewayServiceImpl.BodyTooLargeException.class, stream::read);
|
||||
}
|
||||
|
||||
/**
|
||||
* A body over the limit is a rejection rather than a failure, so it must be distinguishable
|
||||
* from a read that genuinely broke: the route is anonymous, and answering every oversized
|
||||
* body with an error trace lets a caller fill the log at will.
|
||||
*/
|
||||
@Test
|
||||
void testBodyOverTheByteLimitIsRejectedNotFailed() {
|
||||
final PushGatewayServiceImpl.BoundedInputStream stream =
|
||||
new PushGatewayServiceImpl.BoundedInputStream(createBody(BODY.repeat(100)), 16);
|
||||
|
||||
final IOException raised = assertThrows(IOException.class, stream::readAllBytes);
|
||||
|
||||
assertInstanceOf(PushGatewayServiceImpl.BodyTooLargeException.class, raised);
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -373,3 +383,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume.
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -60,6 +60,10 @@ resourceRole:
|
||||
- /api/collector/**===post===[admin,user]
|
||||
- /api/collector/**===put===[admin,user]
|
||||
- /api/collector/**===delete===[admin]
|
||||
- /api/plugin/**===get===[admin]
|
||||
- /api/plugin/**===post===[admin]
|
||||
- /api/plugin/**===put===[admin]
|
||||
- /api/plugin/**===delete===[admin]
|
||||
# the secret config holds the jwt signing key and the aes key protecting stored
|
||||
# credentials, so it stays admin only and is additionally refused by the controller
|
||||
- /api/config/secret===get===[admin]
|
||||
@@ -70,6 +74,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -99,11 +120,18 @@ resourceRole:
|
||||
- /api/account/token===get===[admin]
|
||||
- /api/account/token/**===post===[admin]
|
||||
- /api/account/token/**===delete===[admin]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -140,10 +168,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.startup.security;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assumptions.assumeTrue;
|
||||
import jakarta.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
/**
|
||||
* Guards the springdoc switches that keep the openapi document off by default.
|
||||
*
|
||||
* <p>Scoping the document to the admin role is only half the story. The document is a map
|
||||
* of every route, http method, parameter and model, so a deployment that has no use for it
|
||||
* should not serve it at all: both switches are off unless a deployment opts in, and the
|
||||
* rbac rules stay as the second line of defence for deployments that do. The swagger ui
|
||||
* page itself is reachable anonymously through the {@code /**}{@code /*.html===get}
|
||||
* exclusion, but it renders nothing until the caller proves it holds the admin role.
|
||||
*/
|
||||
class OpenApiDocumentDisabledByDefaultTest {
|
||||
|
||||
private static final Path SCRIPT_DIR = Path.of("..", "script");
|
||||
|
||||
private static final List<List<String>> SWITCHES = List.of(
|
||||
List.of("springdoc", "api-docs", "enabled"),
|
||||
List.of("springdoc", "swagger-ui", "enabled"));
|
||||
|
||||
@Test
|
||||
void shouldDisableTheOpenApiDocumentInThePackagedConfig() throws IOException {
|
||||
try (InputStream in = OpenApiDocumentDisabledByDefaultTest.class.getResourceAsStream("/application.yml")) {
|
||||
assertNotNull(in, "application.yml must be on the classpath");
|
||||
assertDisabled(documentsOf(in), "application.yml");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The deployment scripts ship their own copies of {@code application.yml} and mount
|
||||
* them over the packaged one, so a switch flipped only in the packaged file would
|
||||
* still leave every container deployment serving the document.
|
||||
*/
|
||||
@Test
|
||||
void shouldDisableTheOpenApiDocumentInDeploymentCopies() throws IOException {
|
||||
for (Path copy : deploymentCopies()) {
|
||||
try (InputStream in = Files.newInputStream(copy)) {
|
||||
assertDisabled(documentsOf(in), copy.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that every declaration of the springdoc switches across a multi document
|
||||
* yaml turns the endpoint off, and that at least one declaration exists - a file that
|
||||
* simply omits them falls back to the springdoc default, which is enabled.
|
||||
*
|
||||
* @param documents the yaml documents, in the order spring applies them
|
||||
* @param source the file the documents came from, for the failure message
|
||||
*/
|
||||
private static void assertDisabled(List<Map<String, Object>> documents, String source) {
|
||||
for (List<String> path : SWITCHES) {
|
||||
List<Object> declarations = documents.stream()
|
||||
.map(document -> valueAt(document, path))
|
||||
.filter(Objects::nonNull)
|
||||
.toList();
|
||||
assertFalse(declarations.isEmpty(),
|
||||
String.join(".", path) + " is unset in " + source + ", so it falls back to enabled");
|
||||
declarations.forEach(declared -> assertEquals(Boolean.FALSE, declared,
|
||||
String.join(".", path) + " is enabled in " + source));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param document the parsed yaml document
|
||||
* @param path the key path to walk, outermost first
|
||||
* @return the value at that path, or null when any segment is missing
|
||||
*/
|
||||
@Nullable
|
||||
private static Object valueAt(Map<String, Object> document, List<String> path) {
|
||||
Object current = document;
|
||||
for (String key : path) {
|
||||
if (!(current instanceof Map<?, ?> map)) {
|
||||
return null;
|
||||
}
|
||||
current = map.get(key);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<Map<String, Object>> documentsOf(InputStream in) {
|
||||
List<Map<String, Object>> documents = new ArrayList<>();
|
||||
for (Object document : new Yaml().loadAll(in)) {
|
||||
if (document instanceof Map<?, ?> map) {
|
||||
documents.add((Map<String, Object>) map);
|
||||
}
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the {@code application.yml} copies shipped by the deployment scripts
|
||||
*/
|
||||
private static Set<Path> deploymentCopies() throws IOException {
|
||||
assumeTrue(Files.isDirectory(SCRIPT_DIR),
|
||||
"running outside the source tree, the packaged file asserted above is all we can see");
|
||||
try (Stream<Path> paths = Files.walk(SCRIPT_DIR)) {
|
||||
Set<Path> copies = paths.filter(path -> path.getFileName().toString().equals("application.yml"))
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
assertFalse(copies.isEmpty(), "expected the deployment scripts to ship application.yml copies");
|
||||
return copies;
|
||||
}
|
||||
}
|
||||
}
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.startup.security;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assumptions.assumeTrue;
|
||||
import com.usthe.sureness.matcher.util.TirePathTree;
|
||||
import jakarta.annotation.Nullable;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
/**
|
||||
* Guards the rbac rules covering the generated openapi document.
|
||||
*
|
||||
* <p>These paths used to sit in {@code excludedResource}. Sureness checks the exclusion
|
||||
* tree before any credential check, so an anonymous request returned the full document:
|
||||
* every route, http method, parameter name and type, and every request and response
|
||||
* model. That is a ready made map of the attack surface, so it is scoped like any other
|
||||
* administrative resource.
|
||||
*/
|
||||
class SurenessOpenApiDocRuleTest {
|
||||
|
||||
private static final String SEPARATOR = "===";
|
||||
|
||||
/**
|
||||
* {@code excludedResource} entries are written as {@code api===method}, but
|
||||
* {@code TirePathTree} only accepts the three segment {@code api===method===roles}
|
||||
* shape and silently drops anything else. {@code DefaultPathRoleMatcher} therefore
|
||||
* appends this marker to every excluded rule before it builds the exclusion tree, and
|
||||
* a test that skips the same step builds an empty tree that matches nothing - every
|
||||
* {@code assertNull} against it then passes no matter what the yaml says.
|
||||
*/
|
||||
private static final String EXCLUDE_ROLE = SEPARATOR + "[exclude]";
|
||||
|
||||
private static final Path SCRIPT_DIR = Path.of("..", "script");
|
||||
|
||||
private static TirePathTree roleTree;
|
||||
|
||||
private static TirePathTree excludeTree;
|
||||
|
||||
@BeforeAll
|
||||
@SuppressWarnings("unchecked")
|
||||
static void loadSurenessConfig() throws IOException {
|
||||
List<String> resourceRole;
|
||||
List<String> excludedResource;
|
||||
try (InputStream in = SurenessOpenApiDocRuleTest.class.getResourceAsStream("/sureness.yml")) {
|
||||
assertNotNull(in, "sureness.yml must be on the classpath");
|
||||
Map<String, Object> document = new Yaml().load(in);
|
||||
resourceRole = (List<String>) document.get("resourceRole");
|
||||
excludedResource = (List<String>) document.get("excludedResource");
|
||||
}
|
||||
assertNotNull(resourceRole, "resourceRole must be present");
|
||||
assertNotNull(excludedResource, "excludedResource must be present");
|
||||
roleTree = new TirePathTree();
|
||||
roleTree.buildTree(new LinkedHashSet<>(resourceRole));
|
||||
excludeTree = excludeTreeOf(excludedResource);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldRestrictTheOpenApiDocumentToAdmin() {
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs"));
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/v2/api-docs"));
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/swagger-resources/configuration/ui"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Springdoc also serves the grouped documents and its own config under the same
|
||||
* prefix; a rule bound to the bare path would leave those anonymous.
|
||||
*/
|
||||
@Test
|
||||
void shouldCoverTheGroupedDocumentsToo() {
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs/swagger-config"));
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs/default"));
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs.yaml/default"));
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code OpenApiWebMvcResource} maps the document twice, at the configured path and at
|
||||
* that path with a {@code .yaml} suffix. The suffixed form is a sibling path segment
|
||||
* rather than a child, so {@code /v3/api-docs/**} does not reach it and it needs its
|
||||
* own rule. Without one the yaml document carries no role requirement at all, which
|
||||
* sureness treats as no restriction for any authenticated caller including guest.
|
||||
*/
|
||||
@Test
|
||||
void shouldRestrictTheYamlDocumentToAdmin() {
|
||||
assertEquals("[admin]", rolesFor(roleTree, "/v3/api-docs.yaml"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldStopTreatingTheOpenApiDocumentAsAnonymous() {
|
||||
assertNull(rolesFor(excludeTree, "/v3/api-docs"));
|
||||
assertNull(rolesFor(excludeTree, "/v3/api-docs.yaml"));
|
||||
assertNull(rolesFor(excludeTree, "/v3/api-docs.yaml/default"));
|
||||
assertNull(rolesFor(excludeTree, "/v3/api-docs/swagger-config"));
|
||||
assertNull(rolesFor(excludeTree, "/v2/api-docs"));
|
||||
assertNull(rolesFor(excludeTree, "/swagger-resources/configuration/ui"));
|
||||
}
|
||||
|
||||
/**
|
||||
* The assertions above are all {@code assertNull}, so they only mean something while
|
||||
* the exclusion tree is capable of matching at all. This pins a path that is meant to
|
||||
* stay anonymous and fails if the tree was built in a shape sureness would have
|
||||
* rejected.
|
||||
*/
|
||||
@Test
|
||||
void shouldKeepMatchingIntentionallyExcludedResources() {
|
||||
assertEquals("[exclude]", rolesFor(excludeTree, "/api/i18n/lang"));
|
||||
}
|
||||
|
||||
/**
|
||||
* The deployment scripts ship their own copies of {@code sureness.yml}; a rule fixed
|
||||
* only in the packaged file would still leave every container deployment serving the
|
||||
* document to anonymous callers.
|
||||
*/
|
||||
@Test
|
||||
void deploymentCopiesRestrictTheOpenApiDocument() throws IOException {
|
||||
for (Path copy : deploymentCopies()) {
|
||||
TirePathTree copyRoleTree = new TirePathTree();
|
||||
copyRoleTree.buildTree(new LinkedHashSet<>(sectionOf(copy, "resourceRole")));
|
||||
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs"),
|
||||
"the openapi document is unruled or over-granted in " + copy);
|
||||
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs.yaml"),
|
||||
"the yaml openapi document is unruled or over-granted in " + copy);
|
||||
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs.yaml/default"),
|
||||
"the grouped yaml openapi document is unruled or over-granted in " + copy);
|
||||
assertEquals("[admin]", rolesFor(copyRoleTree, "/v3/api-docs/swagger-config"),
|
||||
"the springdoc ui config is unruled or over-granted in " + copy);
|
||||
|
||||
TirePathTree copyExcludeTree = excludeTreeOf(sectionOf(copy, "excludedResource"));
|
||||
assertNull(rolesFor(copyExcludeTree, "/v3/api-docs"),
|
||||
"the openapi document is still anonymous in " + copy);
|
||||
assertEquals("[exclude]", rolesFor(copyExcludeTree, "/api/i18n/lang"),
|
||||
"the exclusion tree matches nothing in " + copy + ", the assertion above proves nothing");
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private static String rolesFor(TirePathTree tree, String path) {
|
||||
return tree.searchPathFilterRoles(path + SEPARATOR + "get");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param excludedResource the {@code excludedResource} rules, as written in the yaml
|
||||
* @return the exclusion tree, built the way {@code DefaultPathRoleMatcher} builds it
|
||||
*/
|
||||
private static TirePathTree excludeTreeOf(List<String> excludedResource) {
|
||||
TirePathTree tree = new TirePathTree();
|
||||
tree.buildTree(excludedResource.stream()
|
||||
.map(rule -> rule + EXCLUDE_ROLE)
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new)));
|
||||
return tree;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the {@code sureness.yml} copies shipped by the deployment scripts
|
||||
*/
|
||||
private static Set<Path> deploymentCopies() throws IOException {
|
||||
assumeTrue(Files.isDirectory(SCRIPT_DIR),
|
||||
"running outside the source tree, the packaged file asserted above is all we can see");
|
||||
try (Stream<Path> paths = Files.walk(SCRIPT_DIR)) {
|
||||
Set<Path> copies = paths.filter(path -> path.getFileName().toString().equals("sureness.yml"))
|
||||
.collect(Collectors.toCollection(LinkedHashSet::new));
|
||||
assertFalse(copies.isEmpty(), "expected the deployment scripts to ship sureness.yml copies");
|
||||
return copies;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<String> sectionOf(Path copy, String section) throws IOException {
|
||||
Map<String, Object> document;
|
||||
try (InputStream in = Files.newInputStream(copy)) {
|
||||
document = new Yaml().load(in);
|
||||
}
|
||||
List<String> rules = (List<String>) document.get(section);
|
||||
assertNotNull(rules, section + " must be present in " + copy);
|
||||
return rules;
|
||||
}
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.startup.security;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import com.usthe.sureness.matcher.util.TirePathTree;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
/**
|
||||
* Guards the routes that carried no rbac rule at all.
|
||||
*
|
||||
* <p>A route absent from {@code sureness.yml} leaves `supportRoles` null, and
|
||||
* `BaseProcessor.authorized` returns early when no role is required, so every one of
|
||||
* these was reachable by any authenticated account including {@code guest}: a raw promql
|
||||
* passthrough to the time series database, log deletion, log and alert injection, label
|
||||
* management and the internal queue metrics of the hertzbeat process.
|
||||
*/
|
||||
class SurenessUnruledEndpointTest {
|
||||
|
||||
private static final String SEPARATOR = "===";
|
||||
|
||||
private static TirePathTree roleTree;
|
||||
|
||||
@BeforeAll
|
||||
@SuppressWarnings("unchecked")
|
||||
static void loadSurenessConfig() throws IOException {
|
||||
List<String> resourceRole;
|
||||
try (InputStream in = SurenessUnruledEndpointTest.class.getResourceAsStream("/sureness.yml")) {
|
||||
assertNotNull(in, "sureness.yml must be on the classpath");
|
||||
Map<String, Object> document = new Yaml().load(in);
|
||||
resourceRole = (List<String>) document.get("resourceRole");
|
||||
}
|
||||
assertNotNull(resourceRole, "resourceRole must be present");
|
||||
roleTree = new TirePathTree();
|
||||
roleTree.buildTree(new LinkedHashSet<>(resourceRole));
|
||||
}
|
||||
|
||||
private static String rolesFor(String path, String method) {
|
||||
return roleTree.searchPathFilterRoles(path + SEPARATOR + method);
|
||||
}
|
||||
|
||||
/**
|
||||
* `PromqlQueryExecutor` forwards the submitted expression verbatim, so this route reads
|
||||
* the whole metric store regardless of which monitors the caller may see.
|
||||
*/
|
||||
@Test
|
||||
void queryingTheWarehouseDirectlyIsRestrictedToAdmin() {
|
||||
assertEquals("[admin]", rolesFor("/api/warehouse/query", "post"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void probingStorageAvailabilityStaysOpenToEveryRole() {
|
||||
assertEquals("[admin,user,guest]", rolesFor("/api/warehouse/storage/status", "get"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void deletingLogsIsRestrictedToAdmin() {
|
||||
assertEquals("[admin]", rolesFor("/api/logs", "delete"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void readingLogsStaysOpenToEveryRole() {
|
||||
assertEquals("[admin,user,guest]", rolesFor("/api/logs/list", "get"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Matches how the sibling ingestion routes `/api/otlp/**` and `/api/logs/ingest/**`
|
||||
* are already scoped, so a low privileged account can no longer forge log records.
|
||||
*/
|
||||
@Test
|
||||
void ingestingOtlpLogsRequiresAtLeastUser() {
|
||||
assertEquals("[admin,user]", rolesFor("/api/logs/otlp/v1/logs", "post"));
|
||||
}
|
||||
|
||||
/**
|
||||
* The prometheus alertmanager webhook injects alerts, which drive notifications.
|
||||
* Scoped like the sibling `/api/alerts/report` route.
|
||||
*/
|
||||
@Test
|
||||
void injectingPrometheusAlertsRequiresAtLeastUser() {
|
||||
assertEquals("[admin,user]", rolesFor("/api/v2/alerts", "post"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void labelWritesFollowTheUsualScoping() {
|
||||
assertEquals("[admin,user,guest]", rolesFor("/api/label", "get"));
|
||||
assertEquals("[admin,user]", rolesFor("/api/label", "post"));
|
||||
assertEquals("[admin,user]", rolesFor("/api/label", "put"));
|
||||
assertEquals("[admin]", rolesFor("/api/label", "delete"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void processQueueMetricsAreRestrictedToAdmin() {
|
||||
assertEquals("[admin]", rolesFor("/api/metrics", "get"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Favourites are stored per account and rendered on the monitor pages, so they stay
|
||||
* reachable by every role even though they sit under the same path prefix.
|
||||
*/
|
||||
@Test
|
||||
void metricFavouritesStayOpenToEveryRole() {
|
||||
assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1", "get"));
|
||||
assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1/cpu", "post"));
|
||||
assertEquals("[admin,user,guest]", rolesFor("/api/metrics/favorite/1/cpu", "delete"));
|
||||
}
|
||||
}
|
||||
+6
-4
@@ -66,6 +66,7 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://";
|
||||
private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')");
|
||||
private static final Pattern NUL_CHAR_PATTERN = Pattern.compile("\\u0000");
|
||||
private static final String INSTANCE_NULL = "''";
|
||||
private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s";
|
||||
private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s";
|
||||
@@ -346,12 +347,13 @@ public class TdEngineDataStorage extends AbstractHistoryDataStorage {
|
||||
}
|
||||
|
||||
private String formatStringValue(String value) {
|
||||
String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
|
||||
// bugfix Argument list too long
|
||||
if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) {
|
||||
// snmp octet strings may carry NUL padding that breaks the insert sql
|
||||
String formatValue = NUL_CHAR_PATTERN.matcher(value).replaceAll("");
|
||||
// truncate the logical value before escaping so the cut cannot split an escape sequence
|
||||
if (formatValue.length() > tableStrColumnDefineMaxLength) {
|
||||
formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength);
|
||||
}
|
||||
return formatValue;
|
||||
return SQL_SPECIAL_STRING_PATTERN.matcher(formatValue).replaceAll("\\\\$0");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
+26
-3
@@ -115,6 +115,25 @@ class TdEngineDataStorageTest {
|
||||
assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"), "Should contain timestamp and value 68.7");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveDataStripsControlCharacters() throws Exception {
|
||||
tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties);
|
||||
setPrivateField(tdEngineDataStorage, "hikariDataSource", mockHikariDataSource);
|
||||
setParentPrivateField(tdEngineDataStorage, "serverAvailable", true);
|
||||
|
||||
// snmp octet strings can carry NUL bytes (issue #1481); tabs are legitimate data and stay
|
||||
CollectRep.MetricsData metricsData = generateMockedMetricsData("Loopback\tInterface 1\u0000");
|
||||
tdEngineDataStorage.saveData(metricsData);
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture());
|
||||
String executedSql = sqlCaptor.getValue();
|
||||
|
||||
assertTrue(executedSql.contains("Loopback\tInterface 1'"), "NUL stripped, tab preserved");
|
||||
assertTrue(executedSql.indexOf('\u0000') < 0, "no raw NUL byte may reach the sql text");
|
||||
assertTrue(!executedSql.contains("\\u0000"), "no textual NUL escape may survive in the labels json");
|
||||
}
|
||||
|
||||
@Test
|
||||
void destroy() {
|
||||
}
|
||||
@@ -146,6 +165,10 @@ class TdEngineDataStorageTest {
|
||||
}
|
||||
|
||||
public static CollectRep.MetricsData generateMockedMetricsData() {
|
||||
return generateMockedMetricsData("test-%server-01");
|
||||
}
|
||||
|
||||
public static CollectRep.MetricsData generateMockedMetricsData(String instanceValue) {
|
||||
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
|
||||
|
||||
when(mockMetricsData.getId()).thenReturn(0L);
|
||||
@@ -156,9 +179,9 @@ class TdEngineDataStorageTest {
|
||||
when(mockMetricsData.getInstance()).thenReturn("test-%server-01");
|
||||
|
||||
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
|
||||
List<String> columnValues = List.of("test-%server-01", "68.7");
|
||||
List<String> columnValues = List.of(instanceValue, "68.7");
|
||||
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
|
||||
when(mockValueRow.getColumns(0)).thenReturn("test-%server-01");
|
||||
when(mockValueRow.getColumns(0)).thenReturn(instanceValue);
|
||||
when(mockValueRow.getColumns(1)).thenReturn("68.7");
|
||||
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
|
||||
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
|
||||
@@ -177,7 +200,7 @@ class TdEngineDataStorageTest {
|
||||
Field instanceArrowField = new Field("instance", instanceFieldType, null);
|
||||
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
|
||||
when(instanceCell.getField()).thenReturn(instanceArrowField);
|
||||
when(instanceCell.getValue()).thenReturn("test-%server-01");
|
||||
when(instanceCell.getValue()).thenReturn(instanceValue);
|
||||
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
|
||||
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
id: etcd
|
||||
title: Monitoring:etcd monitoring
|
||||
sidebar_label: etcd
|
||||
keywords: [open source monitoring tool, open source middleware monitoring tool, monitoring etcd metrics]
|
||||
---
|
||||
|
||||
> HertzBeat monitors the etcd key-value store by collecting metrics from the Prometheus metrics endpoint that etcd exposes.
|
||||
>
|
||||
> etcd 3.4+ is supported (the database size metric `etcd_mvcc_db_total_size_in_bytes` replaced the old `etcd_debugging_*` name in 3.4).
|
||||
|
||||
## PreRequisites
|
||||
|
||||
### Make sure HertzBeat can reach etcd's metrics endpoint
|
||||
|
||||
etcd exposes Prometheus-format metrics on its client port (default `2379`) at the `/metrics` path. Make sure this address is reachable from HertzBeat:
|
||||
|
||||
1. If etcd only listens on localhost, or client mutual TLS is enabled on the client port, configure a dedicated metrics listener via [`--listen-metrics-urls`](https://etcd.io/docs/latest/op-guide/configuration/). It serves the metrics and health-check endpoints; if exposed without TLS, restrict it to a trusted network.
|
||||
2. Access `{metrics-host}:{metrics-port}/metrics` (the client port `2379` by default) from the HertzBeat host to confirm metrics data can be fetched.
|
||||
|
||||
More information see [etcd monitoring documentation](https://etcd.io/docs/latest/op-guide/monitoring/).
|
||||
|
||||
### Configuration parameter
|
||||
|
||||
| Parameter name | Parameter help description |
|
||||
|----------------------|---------------------------------------------------------------------------------------------|
|
||||
| Target Host | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://) |
|
||||
| Port | Port of the etcd metrics endpoint, default 2379 when using the client listener |
|
||||
| Query Timeout | HTTP request timeout in milliseconds, default 6000 |
|
||||
| HTTPS | Whether to use HTTPS to request the metrics endpoint |
|
||||
| Headers | Optional extra HTTP request headers |
|
||||
| Auth Type | Optional Basic/Digest auth if the metrics endpoint sits behind an auth proxy |
|
||||
| Username / Password | Credentials used when Auth Type is set |
|
||||
|
||||
### Collection Metric
|
||||
|
||||
#### Metric set:etcd_server_has_leader
|
||||
|
||||
| Metric name | Metric unit | Metric help description |
|
||||
|-------------|-------------|---------------------------------------------------------------|
|
||||
| hasLeader | none | Whether this etcd member has a raft leader (1=yes, 0=no) |
|
||||
|
||||
#### Metric set:etcd_mvcc_db_total_size_in_bytes
|
||||
|
||||
| Metric name | Metric unit | Metric help description |
|
||||
|-------------|-------------|--------------------------------------------------------|
|
||||
| dbSize | MB | Total size of the underlying database physically allocated |
|
||||
|
||||
#### Metric set:etcd_server_leader_changes_seen_total
|
||||
|
||||
| Metric name | Metric unit | Metric help description |
|
||||
|-----------------|-------------|-------------------------------------------|
|
||||
| leaderChanges | none | Total number of leader changes observed |
|
||||
|
||||
#### Metric set:process_cpu_seconds_total
|
||||
|
||||
| Metric name | Metric unit | Metric help description |
|
||||
|-------------|-------------|---------------------------------------------------|
|
||||
| cpuSeconds | second | Cumulative user and system CPU time consumed |
|
||||
|
||||
#### Metric set:process_resident_memory_bytes
|
||||
|
||||
| Metric name | Metric unit | Metric help description |
|
||||
|-------------|-------------|-----------------------------------|
|
||||
| memory | MB | Resident memory size of the process |
|
||||
@@ -51,6 +51,13 @@ resourceRole:
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
- /api/status/page/**===delete===[admin]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
@@ -82,10 +89,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
@@ -140,6 +143,32 @@ account:
|
||||
role: [user]
|
||||
```
|
||||
|
||||
## OpenAPI Document And Swagger UI
|
||||
|
||||
The generated OpenAPI document lists every route, http method, parameter name and type, and every request and response model. It is a ready made map of the attack surface, so HertzBeat does not serve it by default: `springdoc.api-docs.enabled` and `springdoc.swagger-ui.enabled` are both `false` in the shipped `application.yml`, which makes `/v3/api-docs` and `/swagger-ui/index.html` return 404.
|
||||
|
||||
If you need the document, opt in by updating the `application.yml` file in the `config` directory:
|
||||
|
||||
```yaml
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: true
|
||||
swagger-ui:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
Once enabled, the document endpoints are still scoped to the `admin` role by the `resourceRole` rules above. Sign in to the HertzBeat web application as an administrator before opening `/swagger-ui/index.html`; the Swagger UI attaches the stored HertzBeat token to its same-origin document and try-it-out requests, and the page loads without asking for anything.
|
||||
|
||||
Without that session the document is not exposed, but the page does not fail silently either. `/swagger-ui/index.html` is a static file and still loads; its request for `/v3/api-docs/swagger-config` is answered with `401` and a `WWW-Authenticate: Digest` challenge, so the browser asks for a username and password. Administrator credentials entered there let the document through, and an account without the `admin` role is answered with `403`.
|
||||
|
||||
The document can also be fetched directly with an administrator token:
|
||||
|
||||
```shell
|
||||
curl -H "Authorization: Bearer $YOUR_ADMIN_TOKEN" http://localhost:1157/v3/api-docs
|
||||
```
|
||||
|
||||
> ⚠️ Do not move the OpenAPI paths into `excludedResource`; doing so makes the complete document anonymous again.
|
||||
|
||||
## Update Security Secret
|
||||
|
||||
> This secret is the key for account security encryption management and needs to be updated to your custom key string of the same length.
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
id: etcd
|
||||
title: 监控:etcd 监控
|
||||
sidebar_label: etcd
|
||||
keywords: [开源监控系统, 中间件监控, etcd监控]
|
||||
---
|
||||
|
||||
> HertzBeat 通过采集 etcd 暴露的 Prometheus metrics 接口数据,对 etcd 键值存储进行监控。
|
||||
>
|
||||
> 支持 etcd 3.4 及以上版本(数据库大小指标 `etcd_mvcc_db_total_size_in_bytes` 自 3.4 起替代旧的 `etcd_debugging_*` 命名)。
|
||||
|
||||
## 监控前操作
|
||||
|
||||
### 确认 HertzBeat 能访问 etcd 的 metrics 接口
|
||||
|
||||
etcd 会在客户端端口(默认 `2379`)的 `/metrics` 路径暴露 Prometheus 格式的指标。请确保 HertzBeat 能访问该地址:
|
||||
|
||||
1. 若 etcd 仅监听 localhost,或客户端端口启用了双向 TLS,请通过 [`--listen-metrics-urls`](https://etcd.io/docs/latest/op-guide/configuration/) 配置独立的 metrics 监听地址。该地址提供 metrics 与健康检查端点;若不加 TLS 暴露,请仅限受信任网络访问。
|
||||
2. 从 HertzBeat 所在机器访问 `{metrics-host}:{metrics-port}/metrics`(默认为客户端端口 `2379`),确认能获取到 metrics 数据。
|
||||
|
||||
更多信息请参考 [etcd 监控文档](https://etcd.io/docs/latest/op-guide/monitoring/)。
|
||||
|
||||
### 配置参数
|
||||
|
||||
| 参数名称 | 参数帮助描述 |
|
||||
|--------|-----------------------------------------------|
|
||||
| 目标Host | 被监控的对端IPV4,IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 |
|
||||
| 端口 | metrics 接口端口,使用客户端 listener 时默认为 2379 |
|
||||
| 查询超时时间 | HTTP请求超时时间,单位毫秒,默认6000 |
|
||||
| 启用HTTPS | 是否使用 HTTPS 请求 metrics 接口 |
|
||||
| 请求Headers | 可选的额外 HTTP 请求头 |
|
||||
| 认证方式 | 若 metrics 接口在认证代理后面,可选 Basic/Digest 认证 |
|
||||
| 用户名/密码 | 配置认证方式后使用的凭据 |
|
||||
|
||||
### 采集指标
|
||||
|
||||
#### 指标集合:etcd_server_has_leader
|
||||
|
||||
| 指标名称 | 指标单位 | 指标帮助描述 |
|
||||
|-----------|------|------------------------------|
|
||||
| hasLeader | 无 | 该 etcd 成员是否存在 raft 领导者(1有0无) |
|
||||
|
||||
#### 指标集合:etcd_mvcc_db_total_size_in_bytes
|
||||
|
||||
| 指标名称 | 指标单位 | 指标帮助描述 |
|
||||
|--------|------|-----------------|
|
||||
| dbSize | MB | 物理分配的数据库总大小 |
|
||||
|
||||
#### 指标集合:etcd_server_leader_changes_seen_total
|
||||
|
||||
| 指标名称 | 指标单位 | 指标帮助描述 |
|
||||
|----------------|------|--------------|
|
||||
| leaderChanges | 无 | 已观测到的领导者变更总次数 |
|
||||
|
||||
#### 指标集合:process_cpu_seconds_total
|
||||
|
||||
| 指标名称 | 指标单位 | 指标帮助描述 |
|
||||
|------------|------|------------------|
|
||||
| cpuSeconds | 秒 | 累计用户与系统CPU使用时间 |
|
||||
|
||||
#### 指标集合:process_resident_memory_bytes
|
||||
|
||||
| 指标名称 | 指标单位 | 指标帮助描述 |
|
||||
|--------|------|---------|
|
||||
| memory | MB | 进程常驻内存大小 |
|
||||
@@ -52,6 +52,13 @@ resourceRole:
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
- /api/status/page/**===delete===[admin]
|
||||
# OpenAPI 文档包含全部路由、参数与数据模型,等同于一份接口地图,
|
||||
# 因此按普通管理类资源收敛到 admin,不再匿名开放
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
|
||||
# 需要被过滤保护的资源,不认证鉴权直接访问
|
||||
# /api/v1/source3===get 表示 /api/v1/source3===get 可以被任何人访问 无需登录认证鉴权
|
||||
@@ -82,10 +89,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
@@ -141,6 +144,32 @@ account:
|
||||
role: [user]
|
||||
```
|
||||
|
||||
## OpenAPI 文档与 Swagger UI
|
||||
|
||||
生成的 OpenAPI 文档会列出全部路由、HTTP 方法、参数名与类型,以及所有请求和响应模型,等同于一份现成的攻击面地图,因此 HertzBeat 默认不对外提供:随包发布的 `application.yml` 中 `springdoc.api-docs.enabled` 与 `springdoc.swagger-ui.enabled` 均为 `false`,此时 `/v3/api-docs` 和 `/swagger-ui/index.html` 返回 404。
|
||||
|
||||
如果确实需要该文档,更新 `config` 目录下的 `application.yml` 文件显式开启:
|
||||
|
||||
```yaml
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: true
|
||||
swagger-ui:
|
||||
enabled: true
|
||||
```
|
||||
|
||||
开启之后,文档接口仍然被上面的 `resourceRole` 规则收敛在 `admin` 角色。请先以管理员身份登录 HertzBeat Web 应用,再打开 `/swagger-ui/index.html`;Swagger UI 会在同源的文档请求与 try-it-out 请求中附带 HertzBeat 保存的令牌,页面不会再要求任何输入。
|
||||
|
||||
没有该会话时文档同样不会泄露,但页面并非静默失败:`/swagger-ui/index.html` 是静态文件,仍然可以打开,而它请求 `/v3/api-docs/swagger-config` 会得到 `401` 与 `WWW-Authenticate: Digest` 挑战,浏览器因此弹出用户名密码框。在该弹框中输入管理员账号可以正常加载文档;没有 `admin` 角色的账号则会收到 `403`。
|
||||
|
||||
也可以直接携带管理员令牌获取文档:
|
||||
|
||||
```shell
|
||||
curl -H "Authorization: Bearer $YOUR_ADMIN_TOKEN" http://localhost:1157/v3/api-docs
|
||||
```
|
||||
|
||||
> ⚠️ 不要把 OpenAPI 路径放回 `excludedResource`,否则完整接口文档会再次匿名开放。
|
||||
|
||||
## 更新安全密钥
|
||||
|
||||
> 此密钥为账户安全加密管理的密钥,需要更新为相同长度的你自定义密钥串。
|
||||
|
||||
@@ -245,6 +245,7 @@
|
||||
"help/kafka_client",
|
||||
"help/pulsar",
|
||||
"help/nacos",
|
||||
"help/etcd",
|
||||
"help/rabbitmq",
|
||||
"help/rocketmq",
|
||||
"help/shenyu",
|
||||
|
||||
@@ -525,9 +525,9 @@ The following components are provided under the MIT License. See project link fo
|
||||
The text of each license is also included in licenses/LICENSE-[project].txt.
|
||||
|
||||
https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
|
||||
https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.21 MIT
|
||||
https://mvnrepository.com/artifact/org.influxdb/influxdb-java/2.23 MIT
|
||||
|
||||
@@ -524,9 +524,9 @@ The following components are provided under the MIT License. See project link fo
|
||||
The text of each license is also included in licenses/LICENSE-[project].txt.
|
||||
|
||||
https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
|
||||
https://mvnrepository.com/artifact/org.codehaus.mojo/animal-sniffer-annotations/1.21 MIT
|
||||
https://mvnrepository.com/artifact/org.influxdb/influxdb-java/2.23 MIT
|
||||
|
||||
@@ -396,9 +396,9 @@ The following components are provided under the MIT License. See project link fo
|
||||
The text of each license is also included in licenses/LICENSE-[project].txt.
|
||||
|
||||
https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc/10.2.0.jre8 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk15on/1.69 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.bouncycastle/bcutil-jdk18on/1.85 MIT
|
||||
https://mvnrepository.com/artifact/org.checkerframework/checker-qual/3.33.0 MIT
|
||||
https://mvnrepository.com/artifact/org.slf4j/jcl-over-slf4j/2.0.9 MIT
|
||||
https://mvnrepository.com/artifact/org.slf4j/jul-to-slf4j/2.0.9 MIT
|
||||
|
||||
@@ -165,6 +165,7 @@
|
||||
<consul-api.version>1.4.5</consul-api.version>
|
||||
<nacos-client.version>3.1.1</nacos-client.version>
|
||||
<vesoft-client.version>3.6.0</vesoft-client.version>
|
||||
<bouncycastle.version>1.85</bouncycastle.version>
|
||||
<jutf7.version>1.0.0</jutf7.version>
|
||||
<huawei.sdk.version>3.1.37</huawei.sdk.version>
|
||||
<huawei.obs.version>3.23.5</huawei.obs.version>
|
||||
|
||||
@@ -71,6 +71,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -373,3 +383,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume.
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -74,6 +74,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -109,7 +119,7 @@ spring:
|
||||
hibernate:
|
||||
format_sql: true
|
||||
dialect: org.hibernate.dialect.MySQLDialect
|
||||
|
||||
|
||||
flyway:
|
||||
enabled: true
|
||||
clean-disabled: true
|
||||
@@ -117,7 +127,7 @@ spring:
|
||||
baseline-version: 1
|
||||
locations:
|
||||
- classpath:db/migration/mysql
|
||||
|
||||
|
||||
# Not Require, Please config if you need email notify
|
||||
mail:
|
||||
# Attention: this is mail server address.
|
||||
@@ -138,7 +148,7 @@ common:
|
||||
queue:
|
||||
# memory or kafka
|
||||
type: memory
|
||||
|
||||
|
||||
warehouse:
|
||||
store:
|
||||
# store history metrics data, enable only one below
|
||||
@@ -222,7 +232,7 @@ alerter:
|
||||
region: AWS_REGION_FOR_END_USER_MESSAGING
|
||||
twilio:
|
||||
account-sid: YOUR_ACCOUNT_SID
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
|
||||
scheduler:
|
||||
server:
|
||||
@@ -273,3 +283,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -70,6 +70,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -92,11 +109,18 @@ resourceRole:
|
||||
- /api/ai/**===delete===[admin]
|
||||
- /api/logs/sse/**===get===[admin,user,guest]
|
||||
- /api/logs/ingest/**===post===[admin,user]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -130,10 +154,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
@@ -74,6 +74,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -109,7 +119,7 @@ spring:
|
||||
hibernate:
|
||||
format_sql: true
|
||||
dialect: org.hibernate.dialect.MySQLDialect
|
||||
|
||||
|
||||
flyway:
|
||||
enabled: true
|
||||
clean-disabled: true
|
||||
@@ -117,7 +127,7 @@ spring:
|
||||
baseline-version: 1
|
||||
locations:
|
||||
- classpath:db/migration/mysql
|
||||
|
||||
|
||||
# Not Require, Please config if you need email notify
|
||||
mail:
|
||||
# Attention: this is mail server address.
|
||||
@@ -138,7 +148,7 @@ common:
|
||||
queue:
|
||||
# memory or kafka
|
||||
type: memory
|
||||
|
||||
|
||||
warehouse:
|
||||
store:
|
||||
# store history metrics data, enable only one below
|
||||
@@ -219,7 +229,7 @@ alerter:
|
||||
region: AWS_REGION_FOR_END_USER_MESSAGING
|
||||
twilio:
|
||||
account-sid: YOUR_ACCOUNT_SID
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
|
||||
scheduler:
|
||||
server:
|
||||
@@ -270,3 +280,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume.
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -70,6 +70,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -92,11 +109,18 @@ resourceRole:
|
||||
- /api/ai/**===delete===[admin]
|
||||
- /api/logs/sse/**===get===[admin,user,guest]
|
||||
- /api/logs/ingest/**===post===[admin,user]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -130,10 +154,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
@@ -74,6 +74,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -109,7 +119,7 @@ spring:
|
||||
hibernate:
|
||||
format_sql: true
|
||||
dialect: org.hibernate.dialect.MySQLDialect
|
||||
|
||||
|
||||
flyway:
|
||||
enabled: true
|
||||
clean-disabled: true
|
||||
@@ -117,7 +127,7 @@ spring:
|
||||
baseline-version: 1
|
||||
locations:
|
||||
- classpath:db/migration/mysql
|
||||
|
||||
|
||||
# Not Require, Please config if you need email notify
|
||||
mail:
|
||||
# Attention: this is mail server address.
|
||||
@@ -142,7 +152,7 @@ warehouse:
|
||||
expire-time: 1h
|
||||
victoria-metrics:
|
||||
enabled: true
|
||||
url: http://victoria-metrics:8428
|
||||
url: http://victoria-metrics:8428
|
||||
username: root
|
||||
password: root
|
||||
insert:
|
||||
@@ -222,7 +232,7 @@ alerter:
|
||||
region: AWS_REGION_FOR_END_USER_MESSAGING
|
||||
twilio:
|
||||
account-sid: YOUR_ACCOUNT_SID
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
|
||||
scheduler:
|
||||
server:
|
||||
@@ -273,3 +283,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume.
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -70,6 +70,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -92,11 +109,18 @@ resourceRole:
|
||||
- /api/ai/**===delete===[admin]
|
||||
- /api/logs/sse/**===get===[admin,user,guest]
|
||||
- /api/logs/ingest/**===post===[admin,user]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -130,10 +154,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
@@ -74,6 +74,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -108,7 +118,7 @@ spring:
|
||||
hibernate:
|
||||
format_sql: true
|
||||
dialect: org.hibernate.dialect.PostgreSQLDialect
|
||||
|
||||
|
||||
flyway:
|
||||
enabled: true
|
||||
clean-disabled: true
|
||||
@@ -116,7 +126,7 @@ spring:
|
||||
baseline-version: 1
|
||||
locations:
|
||||
- classpath:db/migration/postgresql
|
||||
|
||||
|
||||
# Not Require, Please config if you need email notify
|
||||
mail:
|
||||
# Attention: this is mail server address.
|
||||
@@ -219,7 +229,7 @@ alerter:
|
||||
region: AWS_REGION_FOR_END_USER_MESSAGING
|
||||
twilio:
|
||||
account-sid: YOUR_ACCOUNT_SID
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
|
||||
scheduler:
|
||||
server:
|
||||
@@ -270,3 +280,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume.
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -70,6 +70,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -96,11 +113,18 @@ resourceRole:
|
||||
- /api/ingestion/otlp/**===get===[admin,user,guest]
|
||||
- /api/logs/**===get===[admin,user,guest]
|
||||
- /api/traces/**===get===[admin,user,guest]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -137,10 +161,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copy this file to .env and set your own strong password before running docker compose up
|
||||
POSTGRES_USER=root
|
||||
POSTGRES_PASSWORD=
|
||||
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
.env
|
||||
@@ -74,6 +74,16 @@ management:
|
||||
export:
|
||||
enabled: true
|
||||
|
||||
# The generated OpenAPI document is a map of every route, HTTP method, parameter
|
||||
# and model, so it is not served by default. A deployment that wants the Swagger
|
||||
# UI opts in by turning both switches on; the document endpoints stay scoped to
|
||||
# the admin role in sureness.yml either way.
|
||||
springdoc:
|
||||
api-docs:
|
||||
enabled: false
|
||||
swagger-ui:
|
||||
enabled: false
|
||||
|
||||
sureness:
|
||||
container: jakarta_servlet
|
||||
auths:
|
||||
@@ -108,7 +118,7 @@ spring:
|
||||
hibernate:
|
||||
format_sql: true
|
||||
dialect: org.hibernate.dialect.PostgreSQLDialect
|
||||
|
||||
|
||||
flyway:
|
||||
enabled: true
|
||||
clean-disabled: true
|
||||
@@ -116,7 +126,7 @@ spring:
|
||||
baseline-version: 1
|
||||
locations:
|
||||
- classpath:db/migration/postgresql
|
||||
|
||||
|
||||
# Not Require, Please config if you need email notify
|
||||
mail:
|
||||
# Attention: this is mail server address.
|
||||
@@ -221,7 +231,7 @@ alerter:
|
||||
region: AWS_REGION_FOR_END_USER_MESSAGING
|
||||
twilio:
|
||||
account-sid: YOUR_ACCOUNT_SID
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
auth-token: YOUR_AUTH_TOKEN
|
||||
twilio-phone-number: YOUR_TWILIO_PHONE_NUMBER
|
||||
scheduler:
|
||||
server:
|
||||
@@ -272,3 +282,8 @@ hertzbeat:
|
||||
concurrency-limit: 256
|
||||
reject-when-limit-reached: true
|
||||
task-termination-timeout: 5000
|
||||
# Bounds on what a single request to the push gateway may consume.
|
||||
push:
|
||||
max-auto-created-monitors: ${HERTZBEAT_PUSH_MAX_AUTO_CREATED_MONITORS:10000}
|
||||
max-body-bytes: ${HERTZBEAT_PUSH_MAX_BODY_BYTES:5242880}
|
||||
max-samples: ${HERTZBEAT_PUSH_MAX_SAMPLES:10000}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -70,6 +70,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -92,11 +109,18 @@ resourceRole:
|
||||
- /api/ai/**===delete===[admin]
|
||||
- /api/logs/sse/**===get===[admin,user,guest]
|
||||
- /api/logs/ingest/**===post===[admin,user]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -130,10 +154,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
@@ -34,8 +34,8 @@ services:
|
||||
ports:
|
||||
- '15432:5432'
|
||||
environment:
|
||||
POSTGRES_USER: root
|
||||
POSTGRES_PASSWORD: 123456
|
||||
POSTGRES_USER: ${POSTGRES_USER:-root}
|
||||
POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD (e.g., in .env)}"
|
||||
TZ: Asia/Shanghai
|
||||
PGDATA: /var/lib/postgresql/data/pgdata
|
||||
volumes:
|
||||
@@ -73,6 +73,8 @@ services:
|
||||
HERTZBEAT_COLLECTOR_MYSQL_QUERY_ENGINE: auto
|
||||
TZ: Asia/Shanghai
|
||||
LANG: zh_CN.UTF-8
|
||||
SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER:-root}
|
||||
SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?Please set POSTGRES_PASSWORD in your .env file}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
+26
-6
@@ -16,7 +16,7 @@
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
@@ -70,6 +70,23 @@ resourceRole:
|
||||
- /api/config/**===post===[admin]
|
||||
- /api/config/**===put===[admin]
|
||||
- /api/config/**===delete===[admin]
|
||||
# queue depth of the hertzbeat process itself, operational data
|
||||
- /api/metrics===get===[admin]
|
||||
# per account metric favourites rendered on the monitor pages
|
||||
- /api/metrics/**===get===[admin,user,guest]
|
||||
- /api/metrics/**===post===[admin,user,guest]
|
||||
- /api/metrics/**===delete===[admin,user,guest]
|
||||
- /api/label/**===get===[admin,user,guest]
|
||||
- /api/label/**===post===[admin,user]
|
||||
- /api/label/**===put===[admin,user]
|
||||
- /api/label/**===delete===[admin]
|
||||
# the storage availability probe is read by every monitor page, while the query
|
||||
# route forwards a raw promql expression straight to the time series database
|
||||
- /api/warehouse/**===get===[admin,user,guest]
|
||||
- /api/warehouse/query===post===[admin]
|
||||
- /api/logs/otlp/**===post===[admin,user]
|
||||
- /api/logs===delete===[admin]
|
||||
- /api/v2/alerts===post===[admin,user]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
@@ -96,11 +113,18 @@ resourceRole:
|
||||
- /api/ingestion/otlp/**===get===[admin,user,guest]
|
||||
- /api/logs/**===get===[admin,user,guest]
|
||||
- /api/traces/**===get===[admin,user,guest]
|
||||
# The OpenAPI document is a map of every route, parameter and model, so it is
|
||||
# scoped like any other administrative resource instead of being anonymous
|
||||
- /v3/api-docs/**===get===[admin]
|
||||
- /v3/api-docs.yaml===get===[admin]
|
||||
- /v3/api-docs.yaml/**===get===[admin]
|
||||
- /v2/api-docs/**===get===[admin]
|
||||
- /swagger-resources/**===get===[admin]
|
||||
# spring boot actuator exposes jvm, http and datasource internals for scraping
|
||||
- /actuator/**===get===[admin]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
@@ -137,10 +161,6 @@ excludedResource:
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
|
||||
@@ -18,8 +18,7 @@ module.exports = function (config) {
|
||||
// the possible options are listed at https://jasmine.github.io/api/edge/Configuration.html
|
||||
// for example, you can disable the random execution with `random: false`
|
||||
// or set a specific seed with `seed: 4321`
|
||||
},
|
||||
clearContext: false // leave Jasmine Spec Runner output visible in browser
|
||||
}
|
||||
},
|
||||
jasmineHtmlReporter: {
|
||||
suppressAll: true // removes the duplicated traces
|
||||
|
||||
@@ -23,12 +23,12 @@
|
||||
"start": "ng serve --proxy-config proxy.conf.json",
|
||||
"build": "npm run ng-high-memory build",
|
||||
"watch": "ng build --watch --configuration development",
|
||||
"test": "ng test",
|
||||
"test": "ng test --watch=false --browsers=ChromeHeadless",
|
||||
"ng-high-memory": "node --max_old_space_size=8000 ./node_modules/@angular/cli/bin/ng",
|
||||
"hmr": "ng s -o --hmr",
|
||||
"analyze": "npm run ng-high-memory build -- --source-map",
|
||||
"analyze:view": "source-map-explorer dist/**/*.js",
|
||||
"test-coverage": "ng test --code-coverage --watch=false",
|
||||
"test-coverage": "ng test --code-coverage --watch=false --browsers=ChromeHeadless",
|
||||
"color-less": "ng-alain-plugin-theme -t=colorLess",
|
||||
"theme": "ng-alain-plugin-theme -t=themeCss",
|
||||
"icon": "ng g ng-alain:plugin icon",
|
||||
|
||||
@@ -71,7 +71,7 @@ describe('Service: I18n', () => {
|
||||
it('should be use default language when the browser language is not in the list', () => {
|
||||
spyOnProperty(navigator, 'languages').and.returnValue(['es-419']);
|
||||
genModule();
|
||||
expect(srv.defaultLang).toBe('zh-CN');
|
||||
expect(srv.defaultLang).toBe('en-US');
|
||||
});
|
||||
|
||||
it('should be trigger notify when changed language', () => {
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
*/
|
||||
|
||||
import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing';
|
||||
import { Component, NgZone } from '@angular/core';
|
||||
import { Component } from '@angular/core';
|
||||
import { ComponentFixture, TestBed, fakeAsync, tick } from '@angular/core/testing';
|
||||
import { By } from '@angular/platform-browser';
|
||||
import { I18NService } from '@core';
|
||||
@@ -45,6 +45,7 @@ describe('SettingDrawerI18nDirective', () => {
|
||||
let i18nService: jasmine.SpyObj<I18NService>;
|
||||
let httpMock: HttpTestingController;
|
||||
let mockTranslations: { [key: string]: string };
|
||||
const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR'];
|
||||
|
||||
const mockI18nData = {
|
||||
'zh-CN': {
|
||||
@@ -102,7 +103,7 @@ describe('SettingDrawerI18nDirective', () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [HttpClientTestingModule],
|
||||
declarations: [TestComponent, SettingDrawerI18nDirective],
|
||||
providers: [{ provide: ALAIN_I18N_TOKEN, useValue: i18nServiceSpy }, NgZone]
|
||||
providers: [{ provide: ALAIN_I18N_TOKEN, useValue: i18nServiceSpy }]
|
||||
}).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(TestComponent);
|
||||
@@ -120,36 +121,33 @@ describe('SettingDrawerI18nDirective', () => {
|
||||
httpMock.verify();
|
||||
});
|
||||
|
||||
it('should create', () => {
|
||||
function loadMappings(): void {
|
||||
fixture.detectChanges();
|
||||
languages.forEach(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`).flush(mockI18nData[lang as keyof typeof mockI18nData]));
|
||||
}
|
||||
|
||||
function destroyFixture(): void {
|
||||
fixture.destroy();
|
||||
tick(2000);
|
||||
}
|
||||
|
||||
it('should create', fakeAsync(() => {
|
||||
loadMappings();
|
||||
expect(directive).toBeTruthy();
|
||||
});
|
||||
destroyFixture();
|
||||
}));
|
||||
|
||||
it('should load mappings from i18n files', fakeAsync(() => {
|
||||
fixture.detectChanges();
|
||||
|
||||
const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR'];
|
||||
const requests = languages.map(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`));
|
||||
|
||||
languages.forEach((lang, index) => {
|
||||
requests[index].flush(mockI18nData[lang as keyof typeof mockI18nData]);
|
||||
});
|
||||
|
||||
loadMappings();
|
||||
tick(100);
|
||||
fixture.detectChanges();
|
||||
|
||||
expect(i18nService.fanyi).toHaveBeenCalled();
|
||||
destroyFixture();
|
||||
}));
|
||||
|
||||
it('should replace Chinese text with translations', fakeAsync(() => {
|
||||
fixture.detectChanges();
|
||||
|
||||
const languages = ['zh-CN', 'en-US', 'ja-JP', 'pt-BR', 'zh-TW', 'ko-KR'];
|
||||
const requests = languages.map(lang => httpMock.expectOne(`./assets/i18n/${lang}.json`));
|
||||
|
||||
languages.forEach((lang, index) => {
|
||||
requests[index].flush(mockI18nData[lang as keyof typeof mockI18nData]);
|
||||
});
|
||||
|
||||
loadMappings();
|
||||
tick(2000);
|
||||
fixture.detectChanges();
|
||||
tick(100);
|
||||
@@ -161,5 +159,6 @@ describe('SettingDrawerI18nDirective', () => {
|
||||
expect(themeColorDiv.textContent).toContain('Theme Color');
|
||||
}
|
||||
}
|
||||
destroyFixture();
|
||||
}));
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertCenterComponent } from './alert-center.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertCenterComponent', () => {
|
||||
let fixture: ComponentFixture<AlertCenterComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [AlertCenterComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertCenterComponent).compileComponents();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertGroupConvergeComponent } from './alert-group-converge.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertConvergeComponent', () => {
|
||||
let fixture: ComponentFixture<AlertGroupConvergeComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [AlertGroupConvergeComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertGroupConvergeComponent).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertGroupConvergeComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertInhibitComponent } from './alert-inhibit.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertInhibitComponent', () => {
|
||||
let fixture: ComponentFixture<AlertInhibitComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AlertInhibitComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertInhibitComponent).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertInhibitComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
@@ -18,6 +18,8 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureStandaloneTest } from '@testing';
|
||||
import { MarkdownModule } from 'ngx-markdown';
|
||||
|
||||
import { AlertIntegrationComponent } from './alert-integration.component';
|
||||
|
||||
@@ -26,9 +28,7 @@ describe('AlertIntegrationComponent', () => {
|
||||
let fixture: ComponentFixture<AlertIntegrationComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AlertIntegrationComponent]
|
||||
}).compileComponents();
|
||||
await configureStandaloneTest(AlertIntegrationComponent, [MarkdownModule.forRoot()]).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertIntegrationComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
@@ -80,6 +80,11 @@ export class AlertIntegrationComponent implements OnInit {
|
||||
name: this.i18nSvc.fanyi('alert.integration.source.tencent'),
|
||||
icon: 'assets/img/integration/tencent.svg'
|
||||
},
|
||||
{
|
||||
id: 'alibabacloud-cms',
|
||||
name: this.i18nSvc.fanyi('alert.integration.source.alibabacloud-cms'),
|
||||
icon: 'assets/img/integration/alibabacloud.svg'
|
||||
},
|
||||
{
|
||||
id: 'alibabacloud-sls',
|
||||
name: this.i18nSvc.fanyi('alert.integration.source.alibabacloud-sls'),
|
||||
|
||||
+2
-3
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertNoticeReceiverComponent } from './alert-notice-receiver.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertNoticeReceiverComponent', () => {
|
||||
let fixture: ComponentFixture<AlertNoticeReceiverComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AlertNoticeReceiverComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertNoticeReceiverComponent).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertNoticeReceiverComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
+2
-3
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertNoticeRuleComponent } from './alert-notice-rule.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertNoticeRuleComponent', () => {
|
||||
let fixture: ComponentFixture<AlertNoticeRuleComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AlertNoticeRuleComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertNoticeRuleComponent).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertNoticeRuleComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
+2
-3
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertNoticeTemplateComponent } from './alert-notice-template.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertNoticeTemplateComponent', () => {
|
||||
let fixture: ComponentFixture<AlertNoticeTemplateComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [AlertNoticeTemplateComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertNoticeTemplateComponent).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertNoticeTemplateComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertNoticeComponent } from './alert-notice.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertNoticeComponent', () => {
|
||||
let fixture: ComponentFixture<AlertNoticeComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [AlertNoticeComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertNoticeComponent).compileComponents();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertSettingComponent } from './alert-setting.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertSettingComponent', () => {
|
||||
let fixture: ComponentFixture<AlertSettingComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [AlertSettingComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertSettingComponent).compileComponents();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { ComponentFixture, TestBed } from '@angular/core/testing';
|
||||
import { configureShallowTest } from '@testing';
|
||||
|
||||
import { AlertSilenceComponent } from './alert-silence.component';
|
||||
|
||||
@@ -26,9 +27,7 @@ describe('AlertSilenceComponent', () => {
|
||||
let fixture: ComponentFixture<AlertSilenceComponent>;
|
||||
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
declarations: [AlertSilenceComponent]
|
||||
}).compileComponents();
|
||||
await configureShallowTest(AlertSilenceComponent).compileComponents();
|
||||
|
||||
fixture = TestBed.createComponent(AlertSilenceComponent);
|
||||
component = fixture.componentInstance;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user