Compare commits

...
Author SHA1 Message Date
Calvin b69612923f [refactor] optimize code 2025-08-23 00:57:19 +08:00
Calvin 22897e4360 [refactor] optimize code 2025-08-23 00:38:52 +08:00
Calvin 8a705c9ced [refactor] update class and method comments 2025-08-20 02:14:50 +08:00
CalvinandCopilot 2276937b72 Update hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/pojo/JobCache.java
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Calvin <zhengqiwei@apache.org>
2025-08-20 00:11:22 +08:00
tomsun28 553bc1edae Merge branch 'master' into refactor_manager 2025-08-19 23:44:29 +08:00
Calvinandtomsun28 344cb25229 [doc] add japanese i18n in app-redis_cluster.yml (#3672)
Co-authored-by: tomsun28 <tomsun28@outlook.com>
2025-08-19 23:30:11 +08:00
DeleiGuo 74d21a7acd [feature] add user password monitoring metrics in oracle monitor (#3674) 2025-08-19 22:08:05 +08:00
Calvin 2d95df374a [refactor] optimize code structure about collector and job within Manager 2025-08-18 21:21:14 +08:00
f315bf9607 [fix] Fixed Grafana visualization integration display issue (#3666)
Co-authored-by: Calvin <zhengqiwei@apache.org>
Co-authored-by: tomsun28 <tomsun28@outlook.com>
2025-08-17 14:51:14 +08:00
810d54bef5 [improve] Optimize the scheduling logic for batch flush tasks (#3660)
Signed-off-by: Cyanty <153884653+Cyanty@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Sherlock Yin <sherlock.yin1994@gmail.com>
Co-authored-by: Calvin <zhengqiwei@apache.org>
2025-08-17 14:24:48 +08:00
DeleiGuoandCalvin 48eafa1382 [Feature] add Apache DolphinScheduler monitoring support (#3656)
Co-authored-by: Calvin <zhengqiwei@apache.org>
2025-08-17 10:58:46 +08:00
Calvinandtomsun28 4b6c1e0f08 [doc] add japanese i18n in app-redis.yml (#3669)
Co-authored-by: tomsun28 <tomsun28@outlook.com>
2025-08-17 09:59:11 +08:00
edfd857030 [GSOC] MCP server setup, authorization, and basic tool support (#3610)
Signed-off-by: Sarthak Arora <f20200060@pilani.bits-pilani.ac.in>
Co-authored-by: Calvin <zhengqiwei@apache.org>
Co-authored-by: Jast <shenghang@apache.org>
Co-authored-by: tomsun28 <tomsun28@outlook.com>
2025-08-16 19:08:13 +08:00
tomsun28andLogic 408a06d3eb [doc] update contribution doc (#3667)
Co-authored-by: Logic <zqr10159@dromara.org>
2025-08-16 16:07:41 +08:00
74 changed files with 4116 additions and 760 deletions
+85
View File
@@ -0,0 +1,85 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat</artifactId>
<version>2.0-SNAPSHOT</version>
</parent>
<artifactId>hertzbeat-ai-agent</artifactId>
<version>${hertzbeat.version}</version>
<properties>
<spring-ai.version>1.0.1</spring-ai.version>
<java.version>17</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-mcp-server-webmvc</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-starter-model-openai</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-common</artifactId>
</dependency>
<dependency>
<groupId>com.usthe.sureness</groupId>
<artifactId>spring-boot3-starter-sureness</artifactId>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-bom</artifactId>
<version>${spring-ai.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.adapters;
import org.springframework.data.domain.Page;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import java.util.List;
/**
* Interface that provides access to monitor information by retrieving monitor data
* through the underlying monitor service.
*/
public interface MonitorServiceAdapter {
Page<Monitor> getMonitors(
List<Long> ids,
String app,
String search,
Byte status,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
String labels
);
}
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.adapters.impl;
import com.usthe.sureness.subject.SubjectSum;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.agent.adapters.MonitorServiceAdapter;
import org.apache.hertzbeat.ai.agent.config.McpContextHolder;
import org.springframework.data.domain.Page;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.support.SpringContextHolder;
import org.springframework.stereotype.Component;
import java.lang.reflect.Method;
import java.util.List;
/**
* Implementation of the MonitorServiceAdapter interface that provides access to monitor information
* through reflection by invoking the underlying monitor service implementation.
*/
@Slf4j
@Component
public class MonitorServiceAdapterImpl implements MonitorServiceAdapter {
@Override
public Page<Monitor> getMonitors(
List<Long> ids,
String app,
String search,
Byte status,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
String labels) {
try {
// Provide default values for all nullable parameters
if (sort == null || sort.trim().isEmpty()) {
sort = "gmtCreate";
}
if (order == null || order.trim().isEmpty()) {
order = "desc";
}
if (pageIndex == null) {
pageIndex = 0;
}
if (pageSize == null) {
pageSize = 8;
}
Object monitorService = null;
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current security subject: {}", subjectSum);
try {
monitorService = SpringContextHolder.getBean("monitorServiceImpl");
} catch (Exception e) {
log.debug("Could not find bean by name 'monitorServiceImpl', trying by class name");
}
assert monitorService != null;
log.debug("MonitorService bean found: {}", monitorService.getClass().getSimpleName());
Method method = monitorService.getClass().getMethod(
"getMonitors",
List.class, String.class, String.class, Byte.class,
String.class, String.class, int.class, int.class, String.class);
@SuppressWarnings("unchecked")
Page<Monitor> result = (Page<Monitor>) method.invoke(
monitorService,
ids, app, search, status, sort, order, pageIndex, pageSize, labels);
log.debug("MonitorServiceAdapter.getMonitors result: {}", result.getContent());
return result;
} catch (NoSuchMethodException e) {
throw new RuntimeException("Method not found: getMonitors", e);
} catch (Exception e) {
log.debug("Failed to invoke getMonitors via adapter", e);
throw new RuntimeException("Failed to invoke getMonitors via adapter", e);
}
}
}
@@ -0,0 +1,246 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.usthe.sureness.mgt.SurenessSecurityManager;
import com.usthe.sureness.subject.SubjectSum;
import io.modelcontextprotocol.spec.McpError;
import io.modelcontextprotocol.spec.McpSchema;
import io.modelcontextprotocol.spec.McpServerSession;
import io.modelcontextprotocol.spec.McpServerTransport;
import io.modelcontextprotocol.spec.McpServerTransportProvider;
import io.modelcontextprotocol.util.Assert;
import java.io.IOException;
import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import jakarta.servlet.http.HttpServletRequest;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.RouterFunctions;
import org.springframework.web.servlet.function.ServerRequest;
import org.springframework.web.servlet.function.ServerResponse;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
/**
* Custom Server-Sent Events transport provider for Model Context Protocol.
*/
@Slf4j
public class CustomSseServerTransport implements McpServerTransportProvider {
private final ObjectMapper objectMapper;
private final String messageEndpoint;
private final String sseEndpoint;
private final String baseUrl;
@Getter
private final RouterFunction<ServerResponse> routerFunction;
@Setter
private McpServerSession.Factory sessionFactory;
private final Map<String, Object> sessionRequest = new HashMap<>();
private final ConcurrentHashMap<String, McpServerSession> sessions;
private volatile boolean isClosing;
public CustomSseServerTransport(ObjectMapper objectMapper, String messageEndpoint) {
this(objectMapper, messageEndpoint, "/sse");
}
public CustomSseServerTransport(ObjectMapper objectMapper, String messageEndpoint, String sseEndpoint) {
this(objectMapper, "", messageEndpoint, sseEndpoint);
}
public CustomSseServerTransport(ObjectMapper objectMapper, String baseUrl, String messageEndpoint, String sseEndpoint) {
this.sessions = new ConcurrentHashMap();
this.isClosing = false;
Assert.notNull(objectMapper, "ObjectMapper must not be null");
Assert.notNull(baseUrl, "Message base URL must not be null");
Assert.notNull(messageEndpoint, "Message endpoint must not be null");
Assert.notNull(sseEndpoint, "SSE endpoint must not be null");
this.objectMapper = objectMapper;
this.baseUrl = baseUrl;
this.messageEndpoint = messageEndpoint;
this.sseEndpoint = sseEndpoint;
this.routerFunction = RouterFunctions.route().GET(this.sseEndpoint, this::handleSseConnection).POST(this.messageEndpoint, this::handleMessage).build();
}
public Mono<Void> notifyClients(String method, Object params) {
if (this.sessions.isEmpty()) {
log.debug("No active sessions to broadcast message to");
return Mono.empty();
} else {
log.debug("Attempting to broadcast message to {} active sessions", this.sessions.size());
return Flux.fromIterable(this.sessions.values())
.flatMap((session) -> session.sendNotification(method, params)
.doOnError((e) -> log.error("Failed to send message to session {}: {}", session.getId(), e.getMessage()))
.onErrorComplete())
.then();
}
}
public Mono<Void> closeGracefully() {
return Flux.fromIterable(this.sessions.values()).doFirst(() -> {
this.isClosing = true;
log.debug("Initiating graceful shutdown with {} active sessions", this.sessions.size());
}).flatMap(McpServerSession::closeGracefully).then().doOnSuccess((v) -> log.debug("Graceful shutdown completed"));
}
private ServerResponse handleSseConnection(ServerRequest request) {
log.debug("Handling SSE connection for request: {}", request);
HttpServletRequest servletRequest = request.servletRequest();
try {
log.debug("Processing SSE connection for servlet request: {}", servletRequest);
log.debug("Authorization header: {}", servletRequest.getHeader("Authorization"));
} catch (Exception e) {
log.error("Authentication failed for SSE connection: {}", e.getMessage());
return ServerResponse.status(HttpStatus.UNAUTHORIZED).body("Unauthorized: " + e.getMessage());
}
if (this.isClosing) {
return ServerResponse.status(HttpStatus.SERVICE_UNAVAILABLE).body("Server is shutting down");
} else {
String sessionId = UUID.randomUUID().toString();
log.debug("Generated session ID for SSE connection: {}", sessionId);
log.debug("Creating new SSE connection for session: {}", sessionId);
return ServerResponse.sse((sseBuilder) -> {
sseBuilder.onComplete(() -> {
log.debug("SSE connection completed for session: {}", sessionId);
this.sessions.remove(sessionId);
});
sseBuilder.onTimeout(() -> {
log.debug("SSE connection timed out for session: {}", sessionId);
this.sessions.remove(sessionId);
});
CustomSseServerTransport.WebMvcMcpSessionTransport sessionTransport = new CustomSseServerTransport.WebMvcMcpSessionTransport(sessionId, sseBuilder);
McpServerSession session = this.sessionFactory.create(sessionTransport);
this.sessionRequest.put(sessionId, request.servletRequest());
this.sessions.put(sessionId, session);
try {
sseBuilder.id(sessionId).event("endpoint").data(this.baseUrl + this.messageEndpoint + "?sessionId=" + sessionId);
} catch (Exception e) {
log.error("Failed to send initial endpoint event: {}", e.getMessage());
sseBuilder.error(e);
}
}, Duration.ZERO);
}
}
private ServerResponse handleMessage(ServerRequest request) {
if (this.isClosing) {
return ServerResponse.status(HttpStatus.SERVICE_UNAVAILABLE).body("Server is shutting down");
} else if (request.param("sessionId").isEmpty()) {
return ServerResponse.badRequest().body(new McpError("Session ID missing in message endpoint"));
} else {
String sessionId = (String) request.param("sessionId").get();
McpServerSession session = (McpServerSession) this.sessions.get(sessionId);
log.debug("Authorization header for message request: {}", request.servletRequest().getHeader("Authorization"));
SubjectSum subject = SurenessSecurityManager.getInstance().checkIn(sessionRequest.get(sessionId));
McpContextHolder.setSubject(subject);
if (session == null) {
return ServerResponse.status(HttpStatus.NOT_FOUND).body(new McpError("Session not found: " + sessionId));
} else {
try {
String body = request.body(String.class);
McpSchema.JSONRPCMessage message = McpSchema.deserializeJsonRpcMessage(this.objectMapper, body);
session.handle(message).block();
return ServerResponse.ok().build();
} catch (IOException | IllegalArgumentException e) {
log.error("Failed to deserialize message: {}", ((Exception) e).getMessage());
return ServerResponse.badRequest().body(new McpError("Invalid message format"));
} catch (Exception e) {
log.error("Error handling message: {}", e.getMessage());
return ServerResponse.status(HttpStatus.INTERNAL_SERVER_ERROR).body(new McpError(e.getMessage()));
}
}
}
}
private class WebMvcMcpSessionTransport implements McpServerTransport {
private final String sessionId;
private final ServerResponse.SseBuilder sseBuilder;
WebMvcMcpSessionTransport(String sessionId, ServerResponse.SseBuilder sseBuilder) {
this.sessionId = sessionId;
this.sseBuilder = sseBuilder;
log.debug("Session transport {} initialized with SSE builder", sessionId);
}
public Mono<Void> sendMessage(McpSchema.JSONRPCMessage message) {
return Mono.fromRunnable(() -> {
try {
String jsonText = CustomSseServerTransport.this.objectMapper.writeValueAsString(message);
this.sseBuilder.id(this.sessionId).event("message").data(jsonText);
log.debug("Message sent to session {}", this.sessionId);
} catch (Exception e) {
log.error("Failed to send message to session {}: {}", this.sessionId, e.getMessage());
this.sseBuilder.error(e);
}
});
}
public <T> T unmarshalFrom(Object data, TypeReference<T> typeRef) {
return (T) CustomSseServerTransport.this.objectMapper.convertValue(data, typeRef);
}
public Mono<Void> closeGracefully() {
return Mono.fromRunnable(() -> {
log.debug("Closing session transport: {}", this.sessionId);
try {
this.sseBuilder.complete();
log.debug("Successfully completed SSE builder for session {}", this.sessionId);
} catch (Exception e) {
log.warn("Failed to complete SSE builder for session {}: {}", this.sessionId, e.getMessage());
}
});
}
public void close() {
try {
this.sseBuilder.complete();
log.debug("Successfully completed SSE builder for session {}", this.sessionId);
} catch (Exception e) {
log.warn("Failed to complete SSE builder for session {}: {}", this.sessionId, e.getMessage());
}
}
}
}
@@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.openai.OpenAiChatModel;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Configuration class for Large Language Model (LLM) settings.
*/
@Configuration
public class LlmConfig {
@Bean
public ChatClient openAiChatClient(OpenAiChatModel chatModel) {
return ChatClient.create(chatModel);
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import com.usthe.sureness.subject.SubjectSum;
import org.springframework.core.NamedInheritableThreadLocal;
/**
* Context holder for AI agent security context.
*/
public final class McpContextHolder {
private static final ThreadLocal<SubjectSum> subjectHolder =
new NamedInheritableThreadLocal<>("MCP Security and User Identification Context");
private McpContextHolder() {}
/**
* Attaches the user's context to the current thread.
*/
public static void setSubject(SubjectSum subject) {
subjectHolder.set(subject);
}
/**
* Retrieves the context from the current thread.
*/
public static SubjectSum getSubject() {
return subjectHolder.get();
}
/**
* Clears the context from the thread to prevent memory leaks.
*/
public static void clear() {
subjectHolder.remove();
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.config;
import org.springframework.stereotype.Component;
/**
* Provider for system prompts used in the AI agent
*/
@Component
public class PromptProvider {
/**
* Static version of the HertzBeat monitoring prompt
*/
public static final String HERTZBEAT_MONITORING_PROMPT = """
You are an AI assistant specialized in monitoring infrastructure and applications with HertzBeat.
Your role is to help users manage and analyze their monitoring data using the available tools.
You have access to the following HertzBeat monitoring tools:
- list_monitors: Query monitor information with flexible filtering and pagination
- add_monitor: Add a new monitor to the system
When users ask questions about their monitoring setup or data, identify which tool would be most helpful
and use it to provide relevant information. Always provide clear explanations of the monitoring data and
suggest next steps or insights based on the results.
For monitoring-related queries:
1. If users want to see their monitors, use list_monitors with appropriate filters
2. If users want to add a new monitor, use add_monitor with the necessary details
3. If the monitoring information shows potential issues, highlight them and suggest troubleshooting steps
For parameters that accept specific values:
- Monitor status values: 0 (no monitor), 1 (usable), 2 (disabled), 9 (all)
- Sort fields typically include: name, host, app, gmtCreate
- Sort order should be 'asc' or 'desc'
Keep responses focused on monitoring topics and HertzBeat capabilities.
If you're unsure about specific monitoring details, ask clarifying questions before using the tools.
""";
}
@@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.controller;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.http.MediaType;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import org.apache.hertzbeat.ai.agent.pojo.dto.ChatRequestContext;
import org.apache.hertzbeat.ai.agent.service.ChatClientProviderService;
/**
* Controller class for handling chat-related HTTP requests.
*/
@RestController
@RequestMapping("/api/chat")
public class ChatController {
private final ChatClientProviderService chatClientProviderService;
@Autowired
public ChatController(@Qualifier("openAiChatClient") ChatClient openAiChatClient,
ChatClientProviderService chatClientProviderService) {
this.chatClientProviderService = chatClientProviderService;
}
/**
* Send a message and get a streaming response
*
* @param context The chat request context containing message and optional
* conversationId
* @return SSE emitter for streaming response
*/
@PostMapping(value = "/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public SseEmitter streamChat(@RequestBody ChatRequestContext context) {
SseEmitter emitter = new SseEmitter();
new Thread(() -> {
try {
String aiResponse = chatClientProviderService.streamChat(context);
emitter.send(aiResponse);
emitter.complete();
} catch (Exception e) {
emitter.completeWithError(e);
}
}).start();
return emitter;
}
}
@@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.controller;
/**
* Controller for managing conversations.
*/
public class ConversationController {
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.dao;
/**
* Data Access Object interface for Conversation entities.
*/
public interface ConversationDao {
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.dao;
/**
* Data Access Object interface for Message entities.
*/
public interface MessageDao {
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.dao;
/**
* Data Access Object interface for UserPreference entities.
*/
public interface UserPreferenceDao {
}
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.pojo.dto;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Chat request context for AI chat endpoint.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class ChatRequestContext {
/**
* The user's message (required)
*/
private String message;
/**
* Optional conversation ID for context
*/
private String conversationId;
}
@@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
/**
* Service interface for agent operations.
*/
public interface AgentService {
}
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
import org.apache.hertzbeat.ai.agent.pojo.dto.ChatRequestContext;
/**
* Service for interacting with LLM providers (like OpenAI, Anthropic, etc.)
*/
public interface ChatClientProviderService {
String complete(String message);
String streamChat(ChatRequestContext context);
}
@@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.util.List;
import java.util.Map;
/**
* Service for managing chat conversations and interactions with LLM providers.
*/
public interface ConversationService {
/**
* Send a message and receive a streaming response
*
* @param message The user's message
* @param conversationId Optional conversation ID for continuing a chat
* @return SseEmitter for streaming the response
*/
SseEmitter streamChat(String message, String conversationId);
/**
* Send a message and get a complete response
*
* @param message The user's message
* @param conversationId Optional conversation ID for continuing a chat
* @return Response object containing the AI's response and conversation metadata
*/
Map<String, Object> chat(String message, String conversationId);
/**
* Get conversation history for a specific conversation
*
* @param conversationId Conversation ID
* @return Conversation data including messages
*/
Map<String, Object> getConversation(String conversationId);
/**
* Get all conversations for the current user
*
* @return List of conversations
*/
List<Map<String, Object>> getAllConversations();
/**
* Delete a conversation
*
* @param conversationId Conversation ID to delete
*/
void deleteConversation(String conversationId);
}
@@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service;
import org.springframework.ai.tool.ToolCallbackProvider;
/**
* Service interface for MCP server operations.
*/
public interface McpServerService {
ToolCallbackProvider hertzbeatTools();
}
@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.apache.hertzbeat.ai.agent.service.AgentService;
import org.springframework.stereotype.Service;
/**
* Implementation of the AgentService interface.
* This service provides functionality for handling AI agent operations.
*/
@Service
public class AgentServiceImpl implements AgentService {
}
@@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.apache.hertzbeat.ai.agent.config.PromptProvider;
import org.apache.hertzbeat.ai.agent.service.ChatClientProviderService;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.agent.pojo.dto.ChatRequestContext;
import org.springframework.ai.chat.client.ChatClient;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
/**
* Implementation of the {@link ChatClientProviderService}.
* Provides functionality to interact with the ChatClient for handling chat
* messages.
*/
@Service
public class ChatClientProviderServiceImpl implements ChatClientProviderService {
private final ChatClient chatClient;
@Qualifier("hertzbeatTools")
@Autowired
private ToolCallbackProvider toolCallbackProvider;
@Autowired
public ChatClientProviderServiceImpl(@Qualifier("openAiChatClient") ChatClient openAiChatClient) {
this.chatClient = openAiChatClient;
}
@Override
public String complete(String message) {
return this.chatClient.prompt()
.user(message)
.call()
.content();
}
@Override
public String streamChat(ChatRequestContext context) {
try {
return this.chatClient.prompt(PromptProvider.HERTZBEAT_MONITORING_PROMPT)
.user(context.getMessage())
.toolCallbacks(toolCallbackProvider)
.call()
.content();
} catch (Exception e) {
return "Error: " + e.getMessage();
}
}
}
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.springframework.stereotype.Service;
/**
* Implementation of the ConversationService interface for managing chat conversations.
*/
@Service
public class ConversationServiceImpl {
}
@@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.service.impl;
import org.apache.hertzbeat.ai.agent.config.CustomSseServerTransport;
import org.apache.hertzbeat.ai.agent.service.McpServerService;
import org.springframework.ai.mcp.server.autoconfigure.McpServerProperties;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.agent.tools.impl.MonitorToolsImpl;
import org.springframework.ai.tool.ToolCallbackProvider;
import org.springframework.ai.tool.method.MethodToolCallbackProvider;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.web.servlet.function.RouterFunction;
import org.springframework.web.servlet.function.ServerResponse;
/**
* Implementation of the McpServerService interface.
* This service provides functionality for handling MCP server operations.
*/
@Service
@Configuration
public class McpServerServiceImpl implements McpServerService {
@Autowired
private MonitorToolsImpl monitorTools;
@Bean
public ToolCallbackProvider hertzbeatTools() {
return MethodToolCallbackProvider.builder().toolObjects(monitorTools).build();
}
/**
* Provides a custom SSE server transport for the MCP server.
*
* @param objectMapper the ObjectMapper instance for JSON serialization
* @param serverProperties the properties for the MCP server configuration
* @return a CustomSseServerTransport instance configured with the provided properties
*/
@Bean
public CustomSseServerTransport webMvcSseServerTransportProvider(
ObjectMapper objectMapper,
McpServerProperties serverProperties
) {
return new CustomSseServerTransport(
objectMapper,
serverProperties.getBaseUrl(),
serverProperties.getSseMessageEndpoint(),
serverProperties.getSseEndpoint()
);
}
/**
* Provides the MCP server transport bean.
*
* @param transport the custom SSE server transport
* @return the MCP server transport instance
*/
@Primary
@Bean
public RouterFunction<ServerResponse> mvcMcpRouterFunction(CustomSseServerTransport transport) {
return transport.getRouterFunction();
}
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools;
/**
* Tools for alert operations
*/
public interface AlertTools {
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools;
/**
* Tools for metrics operations
*/
public interface MetricsTools {
}
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools;
import org.springframework.ai.chat.model.ToolContext;
import java.util.List;
/**
* Interface for Monitoring Tools
*/
public interface MonitorTools {
String addMonitor(String name, ToolContext context);
/**
* Query monitor information with flexible filtering and pagination.
* Supports filtering by monitor IDs, type, status, host, labels, sorting, and
* pagination.
* Returns results as plain JSON.
*/
String listMonitors(
List<Long> ids,
String app,
Byte status,
String search,
String labels,
String sort,
String order,
Integer pageIndex,
Integer pageSize,
ToolContext context);
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools.impl;
/**
* Implementation of Alert Tools functionality
*/
public class AlertToolsImpl {
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools.impl;
/**
* Implementation of Metrics Tools functionality
*/
public class MetricsToolsImpl {
}
@@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.agent.tools.impl;
import com.usthe.sureness.subject.SubjectSum;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.ai.agent.adapters.MonitorServiceAdapter;
import org.apache.hertzbeat.ai.agent.config.McpContextHolder;
import org.springframework.ai.chat.model.ToolContext;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.data.domain.Page;
import org.springframework.stereotype.Service;
import org.apache.hertzbeat.ai.agent.tools.MonitorTools;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import java.util.List;
/**
* Implementation of Monitoring Tools functionality
*/
@Slf4j
@Service
public class MonitorToolsImpl implements MonitorTools {
@Autowired
private MonitorServiceAdapter monitorServiceAdapter;
/**
* Tool to query monitor information with flexible filtering and pagination.
* Supports filtering by monitor IDs, type, status, host, labels, sorting, and
* pagination.
* Returns monitor names as string.
*/
@Override
@Tool(name = "list_monitors", returnDirect = true, description = """
Query monitor information with flexible filtering and pagination.
Supports filtering by monitor IDs, type, status, host, labels, sorting, and pagination.
Returns results as String. When no parameters are available, pass the default value as mentioned below. If the user doesn't provide any specific parameter, the default value will be used.
""")
public String listMonitors(
@ToolParam(description = "List of monitor IDs to filter (default: empty list)", required = false) List<Long> ids,
@ToolParam(description = "Monitor type, e.g., 'linux' (default: null)", required = false) String app,
@ToolParam(description = "Monitor status (0: no monitor, 1: usable, 2: disabled, 9: all) (default: null)", required = false) Byte status,
@ToolParam(description = "Fuzzy search for host or name (default: null)", required = false) String search,
@ToolParam(description = "Monitor labels, e.g., 'env:prod,instance:22' (default: null)", required = false) String labels,
@ToolParam(description = "Sort field, e.g., 'name' (default: gmtCreate)", required = false) String sort,
@ToolParam(description = "Sort order, 'asc' or 'desc' (default: desc)", required = false) String order,
@ToolParam(description = "Page index (default: 0)", required = false) Integer pageIndex,
@ToolParam(description = "Page size (default: 8)", required = false) Integer pageSize,
ToolContext context) {
try {
Page<Monitor> result = monitorServiceAdapter.getMonitors(ids, app, search, status, sort, order, pageIndex, pageSize, labels);
log.debug("MonitorServiceAdapter.getMonitors result: {}", result);
return result.getContent().stream().map(Monitor::getName).toList().toString();
} catch (Exception e) {
return "error is" + e.getMessage();
}
}
@Override
@Tool(name = "add_monitor", description = "Add a new monitor")
public String addMonitor(@ToolParam(description = "Name of the monitor") String name, ToolContext context) {
log.debug("Adding monitor with name: {}", name);
SubjectSum subjectSum = McpContextHolder.getSubject();
log.debug("Current subject in tool: {}", subjectSum);
return "Monitor added: " + name;
}
}
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.constants;
/**
* Enum representing the possible statuses of a collector.
*/
public enum CollectorStatus {
ONLINE, OFFLINE;
}
@@ -128,11 +128,7 @@ public class DashboardService {
? GrafanaConstants.generateUseDatasource(currentDatasourceName) : "";
String relativeDashboardUrl = grafanaDashboard.getUrl();
if (relativeDashboardUrl != null && grafanaProperties.getUrl() != null && relativeDashboardUrl.startsWith(grafanaProperties.getUrl())) {
relativeDashboardUrl = relativeDashboardUrl.substring(grafanaProperties.getUrl().length());
}
String fullDashboardUrl = grafanaProperties.exposeUrl().replaceAll("/$", "")
+ (relativeDashboardUrl != null ? relativeDashboardUrl.replaceAll("^/", "") : "");
String fullDashboardUrl = grafanaProperties.exposeUrl().replaceAll("/$", "") + relativeDashboardUrl;
grafanaDashboard.setUrl(fullDashboardUrl + KIOSK + REFRESH + INSTANCE + monitorId + useDatasource);
@@ -0,0 +1,141 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.grafana.service;
import org.apache.hertzbeat.common.entity.grafana.GrafanaDashboard;
import org.apache.hertzbeat.grafana.config.GrafanaProperties;
import org.apache.hertzbeat.grafana.dao.DashboardDao;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Test case for {@link DashboardService}
*/
@ExtendWith(MockitoExtension.class)
public class DashboardServiceTest {
@Mock
private ServiceAccountService serviceAccountService;
@Mock
private DashboardDao dashboardDao;
@Mock
private GrafanaProperties grafanaProperties;
@Mock
private RestTemplate restTemplate;
@Mock
private DatasourceService datasourceService;
@InjectMocks
private DashboardService dashboardService;
static final String GRAFANA_API_RESULT = "{"
+ " \"folderUid\": \"\","
+ " \"id\": 3,"
+ " \"slug\": \"prometheus-dashboard\","
+ " \"status\": \"success\","
+ " \"uid\": \"5d7d89b0-b273-40fe-bb30-d652b82f47eb\","
+ " \"url\": \"/d/5d7d89b0-b273-40fe-bb30-d652b82f47eb/prometheus-dashboard\","
+ " \"version\": 3"
+ "}";
@BeforeEach
void setUp() {
when(datasourceService.getCurrentDatasourceName()).thenReturn("hertzbeat-vm-localhost-8428");
when(grafanaProperties.enabled()).thenReturn(Boolean.TRUE);
when(grafanaProperties.getPrefix()).thenReturn("");
when(grafanaProperties.getUrl()).thenReturn("http://127.0.0.1:3000");
when(grafanaProperties.exposeUrl()).thenReturn("http://127.0.0.1:3000");
when(serviceAccountService.getToken()).thenReturn("test-token");
}
@Test
void testCreateOrUpdateDashboard() {
ResponseEntity<String> responseEntity = new ResponseEntity<>(GRAFANA_API_RESULT, HttpStatus.OK);
when(restTemplate.postForEntity(
eq("http://127.0.0.1:3000/api/dashboards/db"), any(HttpEntity.class), eq(String.class)
)).thenReturn(responseEntity);
ArgumentCaptor<GrafanaDashboard> dashboardCaptor = ArgumentCaptor.forClass(GrafanaDashboard.class);
dashboardService.createOrUpdateDashboard("{\"id\":11}", 1L);
verify(dashboardDao).save(dashboardCaptor.capture());
GrafanaDashboard savedDashboard = dashboardCaptor.getValue();
assertNotNull(savedDashboard);
assertNotNull(savedDashboard.getUrl());
String expectedBaseUrl = "http://127.0.0.1:3000/d/5d7d89b0-b273-40fe-bb30-d652b82f47eb";
// Verify that the URL begins with the expected base URL.
assertTrue(savedDashboard.getUrl().startsWith(expectedBaseUrl), "URL should start with: " + expectedBaseUrl + ", but was: " + savedDashboard.getUrl());
assertTrue(savedDashboard.getUrl().contains("kiosk=tv"), "URL should contain kiosk parameter");
assertTrue(savedDashboard.getUrl().contains("refresh=15s"), "URL should contain refresh parameter");
assertTrue(savedDashboard.getUrl().contains("var-instance=1"), "URL should contain instance parameter");
}
@Test
void testCreateOrUpdateDashboardWithTrailingSlash() {
when(grafanaProperties.exposeUrl()).thenReturn("http://127.0.0.1:3000/");
ResponseEntity<String> responseEntity = new ResponseEntity<>(GRAFANA_API_RESULT, HttpStatus.OK);
when(restTemplate.postForEntity(
eq("http://127.0.0.1:3000/api/dashboards/db"), any(HttpEntity.class), eq(String.class)
)).thenReturn(responseEntity);
ArgumentCaptor<GrafanaDashboard> dashboardCaptor = ArgumentCaptor.forClass(GrafanaDashboard.class);
dashboardService.createOrUpdateDashboard("{\"id\":11}", 1L);
verify(dashboardDao).save(dashboardCaptor.capture());
GrafanaDashboard savedDashboard = dashboardCaptor.getValue();
assertNotNull(savedDashboard);
assertNotNull(savedDashboard.getUrl());
String expectedBaseUrl = "http://127.0.0.1:3000/d/5d7d89b0-b273-40fe-bb30-d652b82f47eb";
// Verify that the URL begins with the expected base URL.
assertTrue(savedDashboard.getUrl().startsWith(expectedBaseUrl), "URL should start with: " + expectedBaseUrl + ", but was: " + savedDashboard.getUrl());
assertTrue(savedDashboard.getUrl().contains("kiosk=tv"), "URL should contain kiosk parameter");
assertTrue(savedDashboard.getUrl().contains("refresh=15s"), "URL should contain refresh parameter");
assertTrue(savedDashboard.getUrl().contains("var-instance=1"), "URL should contain instance parameter");
}
}
+5
View File
@@ -210,6 +210,11 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-netty</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-ai-agent</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
@@ -18,6 +18,8 @@
package org.apache.hertzbeat.manager;
import javax.annotation.PostConstruct;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.manager.nativex.HertzbeatRuntimeHintsRegistrar;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -36,10 +38,10 @@ import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableJpaAuditing
@EnableJpaRepositories(basePackages = {"org.apache.hertzbeat"})
@EntityScan(basePackages = {"org.apache.hertzbeat"})
@ComponentScan(basePackages = {"org.apache.hertzbeat"})
@ConfigurationPropertiesScan(basePackages = {"org.apache.hertzbeat"})
@EnableJpaRepositories(basePackages = {ConfigConstants.PkgConstant.PKG})
@EntityScan(basePackages = {ConfigConstants.PkgConstant.PKG})
@ComponentScan(basePackages = {ConfigConstants.PkgConstant.PKG})
@ConfigurationPropertiesScan(basePackages = {ConfigConstants.PkgConstant.PKG})
@ImportRuntimeHints(HertzbeatRuntimeHintsRegistrar.class)
@EnableAsync
@EnableScheduling
@@ -52,4 +54,4 @@ public class Manager {
public void init() {
System.setProperty("jdk.jndi.object.factoriesFilter", "!com.zaxxer.hikari.HikariJNDIFactory");
}
}
}
@@ -40,7 +40,7 @@ import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.manager.StatusPageComponent;
import org.apache.hertzbeat.common.entity.manager.StatusPageHistory;
import org.apache.hertzbeat.common.entity.manager.StatusPageOrg;
import org.apache.hertzbeat.manager.config.StatusProperties;
import org.apache.hertzbeat.manager.properties.StatusProperties;
import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.StatusPageComponentDao;
import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao;
@@ -18,8 +18,8 @@
package org.apache.hertzbeat.manager.config;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.manager.scheduler.ConsistentHash;
import org.apache.hertzbeat.manager.scheduler.SchedulerProperties;
import org.apache.hertzbeat.manager.scheduler.ConsistentHashCollectorKeeper;
import org.apache.hertzbeat.manager.properties.SchedulerProperties;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -33,8 +33,8 @@ import org.springframework.context.annotation.Configuration;
public class SchedulerConfig {
@Bean
public ConsistentHash consistentHasInstance() {
return new ConsistentHash();
public ConsistentHashCollectorKeeper consistentHasInstance() {
return new ConsistentHashCollectorKeeper();
}
}
@@ -19,7 +19,6 @@
package org.apache.hertzbeat.manager.nativex;
import java.lang.reflect.Constructor;
import java.util.Set;
import org.apache.sshd.common.channel.ChannelListener;
import org.apache.sshd.common.forward.PortForwardingEventListener;
@@ -28,7 +27,6 @@ import org.apache.sshd.common.io.nio2.Nio2ServiceFactoryFactory;
import org.apache.sshd.common.session.SessionListener;
import org.apache.sshd.common.util.security.bouncycastle.BouncyCastleSecurityProviderRegistrar;
import org.apache.sshd.common.util.security.eddsa.EdDSASecurityProviderRegistrar;
import org.springframework.aot.hint.ExecutableMode;
import org.springframework.aot.hint.MemberCategory;
import org.springframework.aot.hint.RuntimeHints;
import org.springframework.aot.hint.RuntimeHintsRegistrar;
@@ -60,11 +58,4 @@ public class HertzbeatRuntimeHintsRegistrar implements RuntimeHintsRegistrar {
TypeReference.of(PortForwardingEventListener.class), TypeReference.of(SessionListener.class));
}
}
private void registerConstructor(RuntimeHints hints, Class<?> clazz) {
Constructor<?>[] declaredConstructors = clazz.getDeclaredConstructors();
for (Constructor<?> declaredConstructor : declaredConstructors) {
hints.reflection().registerConstructor(declaredConstructor, ExecutableMode.INVOKE);
}
}
}
@@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.pojo;
import lombok.Data;
import org.apache.hertzbeat.common.constants.CollectorStatus;
import org.apache.hertzbeat.manager.scheduler.AssignJobs;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
* Collector Node
*/
@Data
public class CollectorNode {
/**
* Default number of VM nodes
*/
private static final byte VIRTUAL_NODE_DEFAULT_SIZE = 10;
/**
* collector identity
*/
private final String identity;
/**
* collector mode: public or private
*/
private String mode;
/**
* ip
*/
private String ip;
/**
* collector On-line time stamp
*/
private long uptime;
/**
* collector's own performance service quality score 0 - 127
* The number of virtual nodes will be calculated based on this service quality score
*
*/
private Byte quality;
private CollectorStatus collectorStatus;
/**
* use this collector's collect job ID list
* jobId,jobVersion
*/
private AssignJobs assignJobs;
/**
* the collection task ID list mapped by each virtual node corresponding to this node
* Long[] [0]-jobId, [1]-dispatchHash
*/
private Map<Integer, Set<Long[]>> virtualNodeMap;
public CollectorNode(String identity, String mode, String ip, long uptime, Byte quality) {
this.identity = identity;
this.mode = mode;
this.ip = ip;
this.uptime = uptime;
this.quality = quality;
assignJobs = new AssignJobs();
virtualNodeMap = new ConcurrentHashMap<>(VIRTUAL_NODE_DEFAULT_SIZE);
}
public synchronized void addJob(Integer virtualNodeKey, Integer dispatchHash, Long jobId, boolean isFlushed) {
if (virtualNodeMap == null) {
virtualNodeMap = new ConcurrentHashMap<>(VIRTUAL_NODE_DEFAULT_SIZE);
}
if (assignJobs == null) {
assignJobs = new AssignJobs();
}
Set<Long[]> virtualNodeJob = virtualNodeMap.computeIfAbsent(virtualNodeKey, k -> ConcurrentHashMap.newKeySet(16));
virtualNodeJob.add(new Long[]{jobId, dispatchHash.longValue()});
if (isFlushed) {
assignJobs.addAssignJob(jobId);
} else {
assignJobs.addAddingJob(jobId);
}
}
/**
* obtain the collection task routed by the specified virtual node according to virtualNodeKey
* @param virtualNodeKey virtualNodeKey
* @return collection task
*/
public Set<Long[]> clearVirtualNodeJobs(Integer virtualNodeKey) {
if (virtualNodeMap == null || virtualNodeMap.isEmpty()) {
return null;
}
Set<Long[]> virtualNodeJobs = virtualNodeMap.remove(virtualNodeKey);
virtualNodeMap.put(virtualNodeKey, ConcurrentHashMap.newKeySet(16));
return virtualNodeJobs;
}
public void addVirtualNodeJobs(Integer virtualHashKey, Set<Long[]> reDispatchJobs) {
if (reDispatchJobs == null) {
return;
}
if (virtualNodeMap == null) {
virtualNodeMap = new ConcurrentHashMap<>(16);
}
virtualNodeMap.computeIfPresent(virtualHashKey, (k, v) -> {
reDispatchJobs.addAll(v);
return v;
});
virtualNodeMap.put(virtualHashKey, reDispatchJobs);
}
public void removeVirtualNodeJob(Long jobId) {
if (jobId == null || virtualNodeMap == null) {
return;
}
for (Set<Long[]> jobSet : virtualNodeMap.values()) {
Optional<Long[]> optional = jobSet.stream().filter(item -> Objects.equals(item[0], jobId)).findFirst();
if (optional.isPresent()) {
jobSet.remove(optional.get());
break;
}
}
}
public void destroy() {
if (assignJobs != null) {
assignJobs.clear();
}
if (virtualNodeMap != null) {
virtualNodeMap.clear();
}
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.pojo;
import org.apache.hertzbeat.common.entity.job.Job;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Utility class for caching {@link Job} objects in memory.
* <p>
* This class provides static methods to store, retrieve, and remove {@code Job} instances
* using a thread-safe {@link ConcurrentHashMap}. It is intended to be used as a simple
* in-memory cache for job data within the manager component.
* <p>
* Usage:
* <pre>
* JobCache.put(job);
* Job job = JobCache.get(jobId);
* JobCache.remove(jobId);
* </pre>
*/
public class JobCache {
private static final Map<Long, Job> jobContentCache = new ConcurrentHashMap<>(16);
public static Job get(Long jobId) {
return jobContentCache.get(jobId);
}
public static void put(Job job) {
jobContentCache.put(job.getId(), job);
}
public static void remove(Long jobId) {
jobContentCache.remove(jobId);
}
}
@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hertzbeat.manager.scheduler;
package org.apache.hertzbeat.manager.properties;
import lombok.Getter;
import lombok.Setter;
@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hertzbeat.manager.config;
package org.apache.hertzbeat.manager.properties;
import lombok.Getter;
import lombok.Setter;
@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap;
@Slf4j
@Data
public class AssignJobs {
private static final Integer DEFAULT_CAPACITY = 16;
/**
* current assign jobIds
@@ -51,10 +52,10 @@ public class AssignJobs {
private Set<Long> pinnedJobs;
public AssignJobs() {
jobs = ConcurrentHashMap.newKeySet(16);
addingJobs = ConcurrentHashMap.newKeySet(16);
removingJobs = ConcurrentHashMap.newKeySet(16);
pinnedJobs = ConcurrentHashMap.newKeySet(16);
jobs = ConcurrentHashMap.newKeySet(DEFAULT_CAPACITY);
addingJobs = ConcurrentHashMap.newKeySet(DEFAULT_CAPACITY);
removingJobs = ConcurrentHashMap.newKeySet(DEFAULT_CAPACITY);
pinnedJobs = ConcurrentHashMap.newKeySet(DEFAULT_CAPACITY);
}
public void addAssignJob(Long jobId) {
@@ -29,11 +29,14 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectJobService;
import org.apache.hertzbeat.collector.dispatch.entrance.internal.CollectResponseEventListener;
import org.apache.hertzbeat.common.constants.CollectorStatus;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.entity.dto.ServerInfo;
@@ -53,6 +56,10 @@ import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.pojo.CollectorNode;
import org.apache.hertzbeat.manager.pojo.JobCache;
import org.apache.hertzbeat.manager.properties.SchedulerProperties;
import org.apache.hertzbeat.manager.scheduler.collector.CollectorKeeper;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
import org.apache.hertzbeat.manager.service.AppService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -65,9 +72,7 @@ import org.springframework.stereotype.Component;
@Component
@AutoConfigureAfter(value = {SchedulerProperties.class})
@Slf4j
public class CollectorJobScheduler implements CollectorScheduling, CollectJobScheduling {
private final Map<Long, Job> jobContentCache = new ConcurrentHashMap<>(16);
public class CollectorJobScheduler implements CollectorOperation, CollectorOperationReceiver, JobOperation {
private final Map<Long, CollectResponseEventListener> eventListeners = new ConcurrentHashMap<>(16);
@@ -77,9 +82,6 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
@Autowired
private CollectorMonitorBindDao collectorMonitorBindDao;
@Autowired
private ConsistentHash consistentHash;
@Autowired
private CollectJobService collectJobService;
@@ -92,6 +94,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
@Autowired
private ParamDao paramDao;
@Autowired
private CollectorKeeper collectorKeeper;
@Setter
private ManageServer manageServer;
@Override
@@ -125,15 +131,18 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
.build();
}
collectorDao.save(collector);
ConsistentHash.Node node = new ConsistentHash.Node(identity, collector.getMode(),
collector.getIp(), System.currentTimeMillis(), null);
consistentHash.addNode(node);
reBalanceCollectorAssignJobs();
CollectorNode node = new CollectorNode(identity, collector.getMode(), collector.getIp(), System.currentTimeMillis(), null);
collectorKeeper.addNode(node);
collectorKeeper.changeStatus(identity, CollectorStatus.ONLINE);
collectorKeeper.rebalanceJobs(this::doRebalanceJobs);
// Read database The fixed collection tasks at this collector are delivered
List<CollectorMonitorBind> binds = collectorMonitorBindDao.findCollectorMonitorBindsByCollector(identity);
if (CollectionUtils.isEmpty(binds)){
return;
}
List<Monitor> monitors = monitorDao.findMonitorsByIdIn(binds.stream().map(CollectorMonitorBind::getMonitorId).collect(Collectors.toSet()));
for (Monitor monitor : monitors) {
if (Objects.isNull(monitor) || monitor.getStatus() == CommonConstants.MONITOR_PAUSED_CODE) {
@@ -189,56 +198,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
}
collector.setStatus(CommonConstants.COLLECTOR_STATUS_OFFLINE);
collectorDao.save(collector);
consistentHash.removeNode(identity);
reBalanceCollectorAssignJobs();
log.info("the collector: {} go offline success.", identity);
}
@Override
public void reBalanceCollectorAssignJobs() {
consistentHash.getAllNodes().entrySet().parallelStream().forEach(entry -> {
String collectorName = entry.getKey();
AssignJobs assignJobs = entry.getValue().getAssignJobs();
if (StringUtils.isBlank(collectorName) || Objects.isNull(assignJobs)) {
return;
}
if (CollectionUtils.isNotEmpty(assignJobs.getAddingJobs())) {
Set<Long> addedJobIds = new HashSet<>(8);
for (Long addingJobId : assignJobs.getAddingJobs()) {
Job job = jobContentCache.get(addingJobId);
if (Objects.isNull(job)) {
log.error("assigning job {} content is null.", addingJobId);
continue;
}
addedJobIds.add(addingJobId);
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
.build();
this.manageServer.sendMsg(collectorName, message);
}
}
assignJobs.addAssignJobs(addedJobIds);
assignJobs.removeAddingJobs(addedJobIds);
}
if (CollectionUtils.isNotEmpty(assignJobs.getRemovingJobs())) {
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
assignJobs.getRemovingJobs().forEach(jobId -> collectJobService.cancelAsyncCollectJob(jobId));
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(assignJobs.getRemovingJobs())))
.build();
this.manageServer.sendMsg(collectorName, message);
}
assignJobs.clearRemovingJobs();
}
});
collectorKeeper.changeStatus(identity, CollectorStatus.OFFLINE);
collectorKeeper.rebalanceJobs(this::doRebalanceJobs);
log.info("the collector: {} go offline success.", identity);
}
@Override
@@ -285,56 +248,11 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
return true;
}
@Override
public List<CollectRep.MetricsData> collectSyncJobData(Job job) {
// todo dispatchKey ip+port or id
String dispatchKey = String.valueOf(job.getMonitorId());
ConsistentHash.Node node = consistentHash.preDispatchJob(dispatchKey);
if (Objects.isNull(node)) {
log.error("there is no collector online to assign job.");
CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder()
.setCode(CollectRep.Code.FAIL)
.setMsg("no collector online to assign job")
.build();
return Collections.singletonList(metricsData);
}
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
return collectJobService.collectSyncJobData(job);
} else {
List<CollectRep.MetricsData> metricsData = new LinkedList<>();
CountDownLatch countDownLatch = new CountDownLatch(1);
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
.build();
boolean result = this.manageServer.sendMsg(node.getIdentity(), message);
if (result) {
CollectResponseEventListener listener = new CollectResponseEventListener() {
@Override
public void response(List<CollectRep.MetricsData> responseMetrics) {
if (responseMetrics != null) {
metricsData.addAll(responseMetrics);
}
countDownLatch.countDown();
}
};
eventListeners.put(job.getMonitorId(), listener);
}
try {
countDownLatch.await(120, TimeUnit.SECONDS);
} catch (Exception e) {
log.info("The sync task runs for 120 seconds with no response and returns");
}
return metricsData;
}
}
@Override
public List<CollectRep.MetricsData> collectSyncJobData(Job job, String collector) {
ConsistentHash.Node node = consistentHash.getNode(collector);
CollectorNode node = StringUtils.isBlank(collector)
? collectorKeeper.determineNode(job.getMonitorId())
: collectorKeeper.getNode(collector);
if (Objects.isNull(node)) {
log.error("there is no collector online to assign job.");
CollectRep.MetricsData metricsData = CollectRep.MetricsData.newBuilder()
@@ -343,9 +261,11 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
.build();
return Collections.singletonList(metricsData);
}
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
return collectJobService.collectSyncJobData(job);
}
List<CollectRep.MetricsData> metricsData = new LinkedList<>();
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.ISSUE_ONE_TIME_TASK)
@@ -378,25 +298,10 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
public long addAsyncCollectJob(Job job, String collector) {
long jobId = SnowFlakeIdGenerator.generateId();
job.setId(jobId);
jobContentCache.put(jobId, job);
ConsistentHash.Node node;
if (StringUtils.isBlank(collector)) {
// todo dispatchKey ip+port or id
String dispatchKey = String.valueOf(job.getMonitorId());
node = consistentHash.dispatchJob(dispatchKey, jobId);
if (node == null) {
log.error("there is no collector online to assign job.");
return jobId;
}
} else {
node = consistentHash.getNode(collector);
if (node == null) {
log.error("there is no collector name: {} online to assign job.", collector);
return jobId;
}
node.getAssignJobs().addPinnedJob(jobId);
}
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
CollectorNode collectorNode = collectorKeeper.addJob(job, collector);
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorNode.getIdentity())) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
@@ -404,27 +309,16 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
.build();
this.manageServer.sendMsg(node.getIdentity(), message);
this.manageServer.sendMsg(collectorNode.getIdentity(), message);
}
return jobId;
}
@Override
public long updateAsyncCollectJob(Job modifyJob) {
// delete and add
long preJobId = modifyJob.getId();
long newJobId = addAsyncCollectJob(modifyJob, null);
jobContentCache.remove(preJobId);
cancelAsyncCollectJob(preJobId);
return newJobId;
}
@Override
public long updateAsyncCollectJob(Job modifyJob, String collector) {
// delete and add
long preJobId = modifyJob.getId();
long newJobId = addAsyncCollectJob(modifyJob, collector);
jobContentCache.remove(preJobId);
cancelAsyncCollectJob(preJobId);
return newJobId;
}
@@ -434,24 +328,21 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
if (jobId == null) {
return;
}
jobContentCache.remove(jobId);
for (ConsistentHash.Node node : consistentHash.getAllNodes().values()) {
AssignJobs assignJobs = node.getAssignJobs();
if (assignJobs.getPinnedJobs().remove(jobId)
|| assignJobs.getJobs().remove(jobId) || assignJobs.getAddingJobs().remove(jobId)) {
node.removeVirtualNodeJob(jobId);
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(node.getIdentity())) {
collectJobService.cancelAsyncCollectJob(jobId);
} else {
ClusterMsg.Message deleteMessage = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(List.of(jobId))))
.build();
this.manageServer.sendMsg(node.getIdentity(), deleteMessage);
}
// break; if is there jod exist in multi collector?
}
CollectorNode collectorNode = collectorKeeper.removeJob(jobId);
if (collectorNode == null) {
return;
}
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorNode.getIdentity())) {
collectJobService.cancelAsyncCollectJob(jobId);
} else {
ClusterMsg.Message deleteMessage = ClusterMsg.Message.newBuilder()
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setDirection(ClusterMsg.Direction.REQUEST)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(List.of(jobId))))
.build();
this.manageServer.sendMsg(collectorNode.getIdentity(), deleteMessage);
}
}
@@ -468,7 +359,55 @@ public class CollectorJobScheduler implements CollectorScheduling, CollectJobSch
}
}
public void setManageServer(ManageServer manageServer) {
this.manageServer = manageServer;
private void doRebalanceJobs(AssignJobs assignJobs, String collectorName) {
handleAddingJobs(assignJobs, collectorName);
handleRemovingJobs(assignJobs, collectorName);
}
private void handleAddingJobs(AssignJobs assignJobs, String collectorName) {
if (CollectionUtils.isEmpty(assignJobs.getAddingJobs())) {
return;
}
Set<Long> addedJobIds = new HashSet<>(8);
for (Long addingJobId : assignJobs.getAddingJobs()) {
Job job = JobCache.get(addingJobId);
if (Objects.isNull(job)) {
log.error("assigning job {} content is null.", addingJobId);
continue;
}
addedJobIds.add(addingJobId);
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
collectJobService.addAsyncCollectJob(job);
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.ISSUE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(job)))
.build();
this.manageServer.sendMsg(collectorName, message);
}
}
assignJobs.addAssignJobs(addedJobIds);
assignJobs.removeAddingJobs(addedJobIds);
}
private void handleRemovingJobs(AssignJobs assignJobs, String collectorName) {
if (CollectionUtils.isEmpty(assignJobs.getRemovingJobs())) {
return;
}
if (CommonConstants.MAIN_COLLECTOR_NODE.equals(collectorName)) {
assignJobs.getRemovingJobs().forEach(jobId -> collectJobService.cancelAsyncCollectJob(jobId));
} else {
ClusterMsg.Message message = ClusterMsg.Message.newBuilder()
.setDirection(ClusterMsg.Direction.REQUEST)
.setType(ClusterMsg.MessageType.DELETE_CYCLIC_TASK)
.setMsg(ByteString.copyFromUtf8(JsonUtil.toJson(assignJobs.getRemovingJobs())))
.build();
this.manageServer.sendMsg(collectorName, message);
}
assignJobs.clearRemovingJobs();
}
}
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.scheduler;
/**
* Interface defining operations for managing collector.
* Implementations of this interface provide functionality to control the operation state
* of collectors in the system.
*/
public interface CollectorOperation {
/**
* Takes a collector offline by stopping its collection operations.
* This is typically used for maintenance, updates, or when the collector is no longer needed.
*
* @param identity The unique identifier of the collector to be taken offline
* @return true if the collector was successfully taken offline,
* false if the operation failed or the collector wasn't found
*/
boolean offlineCollector(String identity);
/**
* Brings a collector online by starting its collection operations.
* This is used to activate a collector that was previously offline.
*
* @param identity The unique identifier of the collector to be brought online
* @return true if the collector was successfully brought online,
* false if the operation failed or the collector wasn't found
*/
boolean onlineCollector(String identity);
}
@@ -20,39 +20,25 @@ package org.apache.hertzbeat.manager.scheduler;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
/**
* slave collector service
* Interface defining operations for receiving collector status updates from remote collectors.
* This interface serves as a callback mechanism for handling collector online/offline events.
*/
public interface CollectorScheduling {
public interface CollectorOperationReceiver {
/**
* register collector go online
* @param identity collector identity name
* @param collectorInfo collector information
* Notifies the system when a collector comes online.
* This method should be called when a collector establishes connection and becomes available.
*
* @param identity The unique identifier of the collector (e.g., hostname, IP, or custom ID)
* @param collectorInfo Detailed information about the collector including capabilities,
* configuration, and status metadata
*/
void collectorGoOnline(String identity, CollectorInfo collectorInfo);
/**
* register collector go offline
* @param identity collector identity name
*/
void collectorGoOffline(String identity);
/**
* reBalance dispatch monitoring jobs when collector go online or offline or timeout
*/
void reBalanceCollectorAssignJobs();
/**
* offline collector(stop collector collect operation)
* @param identity collector identity name
* @return true/false
*/
boolean offlineCollector(String identity);
/**
* online collector(start collector collect operation)
* @param identity collector identity name
* @return true/false
* Notifies the system when a collector goes offline.
* This method should be called when a collector disconnects or becomes unavailable.
*
* @param identity The unique identifier of the collector to be marked as offline
*/
boolean onlineCollector(String identity);
void collectorGoOffline(String identity);
}
@@ -23,46 +23,149 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.constants.CollectorStatus;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.manager.pojo.CollectorNode;
import org.apache.hertzbeat.manager.pojo.JobCache;
import org.apache.hertzbeat.manager.scheduler.collector.CollectorKeeper;
/**
* Collector and task mapping scheduling implemented by consistent hashing
*/
@Slf4j
public class ConsistentHash {
public class ConsistentHashCollectorKeeper implements CollectorKeeper {
/**
* consistent hash circle
*/
private final ConcurrentTreeMap<Integer, Node> hashCircle;
private final ConcurrentTreeMap<Integer, CollectorNode> hashCircle = new ConcurrentTreeMap<>();
/**
* collector node
*/
private final Map<String, Node> existNodeMap;
private final Map<String, CollectorNode> existNodeMap = new ConcurrentHashMap<>(16);
/**
* not dispatched job cache
* not dispatched job cache, in order to obtain the cached collection scheduling task
*/
private final List<DispatchJob> dispatchJobCache;
@Getter
private final List<DispatchJob> dispatchJobCache = Collections.synchronizedList(new LinkedList<>());
/**
* Default number of VM nodes
*/
private static final byte VIRTUAL_NODE_DEFAULT_SIZE = 10;
public ConsistentHash() {
hashCircle = new ConcurrentTreeMap<>();
existNodeMap = new ConcurrentHashMap<>(16);
dispatchJobCache = Collections.synchronizedList(new LinkedList<>());
/**
* add collector node
* @param newNode node
*/
@Override
public void addNode(CollectorNode newNode) {
// when mode is cluster public, need reBalance dispatch jobs. else not when is cloud-edge private
if (!CommonConstants.MODE_PRIVATE.equals(newNode.getMode())) {
byte virtualNodeNum = newNode.getQuality() == null ? VIRTUAL_NODE_DEFAULT_SIZE : newNode.getQuality();
for (byte i = 0; i < virtualNodeNum; i++) {
addVirtualNode(newNode, newNode.getIdentity() + i);
}
}
existNodeMap.put(newNode.getIdentity(), newNode);
dispatchJobInCache();
}
@Override
public CollectorNode addJob(Job job, String collectorId) {
JobCache.put(job);
CollectorNode collectorNode;
if (StringUtils.isBlank(collectorId)) {
// todo dispatchKey ip+port or id
String dispatchKey = String.valueOf(job.getMonitorId());
collectorNode = this.dispatchJob(dispatchKey, job.getId());
if (collectorNode == null) {
log.error("there is no collector online to assign job.");
}
} else {
collectorNode = getNode(collectorId);
if (collectorNode == null) {
log.error("there is no collector name: {} online to assign job.", collectorId);
return null;
}
collectorNode.getAssignJobs().addPinnedJob(job.getId());
}
return collectorNode;
}
/**
* get node
* @param collectorName collector name
* @return node
*/
@Override
public CollectorNode getNode(String collectorName) {
return existNodeMap.get(collectorName);
}
@Override
public CollectorNode determineNode(Long jobId) {
String dispatchKey = String.valueOf(jobId);
if (dispatchKey == null || StringUtils.isBlank(dispatchKey)) {
log.error("The dispatch key can not null.");
return null;
}
int dispatchHash = hash(dispatchKey);
return preDispatchJob(dispatchHash);
}
@Override
public void changeStatus(String collectorId, CollectorStatus collectorStatus) {
switch (collectorStatus) {
case ONLINE -> this.getNode(collectorId).setCollectorStatus(collectorStatus);
case OFFLINE -> this.removeNode(collectorId);
default -> {}
}
}
@Override
public void rebalanceJobs(BiConsumer<AssignJobs, String> assignJobCollectorConsumer) {
existNodeMap.entrySet().parallelStream().forEach(entry -> {
String collectorName = entry.getKey();
AssignJobs assignJobs = entry.getValue().getAssignJobs();
if (StringUtils.isBlank(collectorName) || Objects.isNull(assignJobs)) {
return;
}
assignJobCollectorConsumer.accept(assignJobs, collectorName);
});
}
@Override
public CollectorNode removeJob(Long jobId) {
JobCache.remove(jobId);
for (CollectorNode node : existNodeMap.values()) {
AssignJobs assignJobs = node.getAssignJobs();
if (assignJobs.getPinnedJobs().remove(jobId)
|| assignJobs.getJobs().remove(jobId) || assignJobs.getAddingJobs().remove(jobId)) {
node.removeVirtualNodeJob(jobId);
return node;
// break; if is there jod exist in multi collector?
}
}
return null;
}
/**
@@ -70,15 +173,15 @@ public class ConsistentHash {
* @param newNode node
* @param identity virtual node identity
*/
public synchronized void addVirtualNode(Node newNode, String identity){
private synchronized void addVirtualNode(CollectorNode newNode, String identity){
int virtualHashKey = hash(identity);
hashCircle.put(virtualHashKey, newNode);
newNode.addVirtualNodeJobs(virtualHashKey, ConcurrentHashMap.newKeySet(16));
Map.Entry<Integer, Node> higherVirtualNode = hashCircle.higherOrFirstEntry(virtualHashKey);
Map.Entry<Integer, CollectorNode> higherVirtualNode = hashCircle.higherOrFirstEntry(virtualHashKey);
// Reassign tasks that are routed to the higherVirtualNode virtual node
// Tasks are either on the original virtual node or on the new virtual node
Integer higherVirtualNodeKey = higherVirtualNode.getKey();
Node higherNode = higherVirtualNode.getValue();
CollectorNode higherNode = higherVirtualNode.getValue();
Set<Long[]> dispatchJobs = higherNode.clearVirtualNodeJobs(higherVirtualNodeKey);
if (dispatchJobs != null && !dispatchJobs.isEmpty()) {
Set<Long[]> reDispatchJobs = ConcurrentHashMap.newKeySet(dispatchJobs.size());
@@ -91,53 +194,37 @@ public class ConsistentHash {
iterator.remove();
}
}
higherNode.virtualNodeMap.put(higherVirtualNodeKey, dispatchJobs);
higherNode.getVirtualNodeMap().put(higherVirtualNodeKey, dispatchJobs);
Set<Long> jobIds = reDispatchJobs.stream().map(item -> item[0]).collect(Collectors.toSet());
newNode.addVirtualNodeJobs(virtualHashKey, reDispatchJobs);
if (higherNode != newNode) {
higherNode.assignJobs.removeAssignJobs(jobIds);
higherNode.assignJobs.addRemovingJobs(jobIds);
newNode.assignJobs.addAddingJobs(jobIds);
higherNode.getAssignJobs().removeAssignJobs(jobIds);
higherNode.getAssignJobs().addRemovingJobs(jobIds);
newNode.getAssignJobs().addAddingJobs(jobIds);
}
}
}
/**
* add collector node
* @param newNode node
*/
public void addNode(Node newNode) {
// when mode is cluster public, need reBalance dispatch jobs. else not when is cloud-edge private
if (!CommonConstants.MODE_PRIVATE.equals(newNode.mode)) {
byte virtualNodeNum = newNode.quality == null ? VIRTUAL_NODE_DEFAULT_SIZE : newNode.quality;
for (byte i = 0; i < virtualNodeNum; i++) {
addVirtualNode(newNode, newNode.identity + i);
}
}
existNodeMap.put(newNode.identity, newNode);
dispatchJobInCache();
}
/**
* remove virtual node
* @param deletedNode node
* @param virtualNodeHash virtual node hash key
*/
public synchronized void removeVirtualNode(Node deletedNode, Integer virtualNodeHash) {
Set<Long[]> removeJobHashSet = deletedNode.virtualNodeMap.get(virtualNodeHash);
private synchronized void removeVirtualNode(CollectorNode deletedNode, Integer virtualNodeHash) {
Set<Long[]> removeJobHashSet = deletedNode.getVirtualNodeMap().get(virtualNodeHash);
// Migrate the virtualNodeEntry collection task to the nearest virtual node that is larger than it
hashCircle.remove(virtualNodeHash);
if (removeJobHashSet == null || removeJobHashSet.isEmpty()) {
return;
}
Map.Entry<Integer, Node> higherVirtualEntry = hashCircle.higherOrFirstEntry(virtualNodeHash);
Map.Entry<Integer, CollectorNode> higherVirtualEntry = hashCircle.higherOrFirstEntry(virtualNodeHash);
if (higherVirtualEntry == null || higherVirtualEntry.getValue() == deletedNode) {
higherVirtualEntry = null;
}
// jobId
Set<Long> removeJobIds = removeJobHashSet.stream().map(item -> item[0]).collect(Collectors.toSet());
deletedNode.assignJobs.removeAssignJobs(removeJobIds);
deletedNode.assignJobs.addRemovingJobs(removeJobIds);
deletedNode.getAssignJobs().removeAssignJobs(removeJobIds);
deletedNode.getAssignJobs().addRemovingJobs(removeJobIds);
if (higherVirtualEntry == null) {
// jobId-dispatchHash
removeJobHashSet.forEach(value -> {
@@ -150,9 +237,9 @@ public class ConsistentHash {
}
});
} else {
Node higherVirtualNode = higherVirtualEntry.getValue();
CollectorNode higherVirtualNode = higherVirtualEntry.getValue();
higherVirtualNode.addVirtualNodeJobs(higherVirtualEntry.getKey(), removeJobHashSet);
higherVirtualNode.assignJobs.addAddingJobs(removeJobIds);
higherVirtualNode.getAssignJobs().addAddingJobs(removeJobIds);
}
}
@@ -160,20 +247,19 @@ public class ConsistentHash {
* deleted collector node
* @param name collector name
*/
public Node removeNode(String name) {
Node deletedNode = existNodeMap.remove(name);
private void removeNode(String name) {
CollectorNode deletedNode = existNodeMap.remove(name);
if (deletedNode == null) {
return null;
}
for (Integer virtualNodeHash : deletedNode.virtualNodeMap.keySet()) {
removeVirtualNode(deletedNode, virtualNodeHash);
return;
}
deletedNode.getVirtualNodeMap().keySet()
.forEach(virtualNodeHash -> removeVirtualNode(deletedNode, virtualNodeHash));
deletedNode.destroy();
dispatchJobInCache();
return deletedNode;
}
public synchronized void dispatchJobInCache() {
private synchronized void dispatchJobInCache() {
if (!dispatchJobCache.isEmpty()) {
int size = dispatchJobCache.size();
for (int index = 0; index < size; index++) {
@@ -183,31 +269,6 @@ public class ConsistentHash {
}
}
/**
* get all collector nodes
* @return nodes
*/
public Map<String, Node> getAllNodes() {
return existNodeMap;
}
/**
* get node
* @param collectorName collector name
* @return node
*/
public Node getNode(String collectorName) {
return existNodeMap.get(collectorName);
}
/**
* Obtain the cached collection scheduling task
* @return cache task
*/
public List<DispatchJob> getDispatchJobCache() {
return dispatchJobCache;
}
/**
* obtain the collector node according to the collection task information
*
@@ -215,7 +276,7 @@ public class ConsistentHash {
* @param jobId jobId
* @return collector node
*/
public Node dispatchJob(String dispatchKey, Long jobId) {
private CollectorNode dispatchJob(String dispatchKey, Long jobId) {
if (dispatchKey == null || StringUtils.isBlank(dispatchKey)) {
log.error("The dispatch key can not null.");
return null;
@@ -224,38 +285,23 @@ public class ConsistentHash {
return dispatchJob(dispatchHash, jobId, true);
}
/**
* The collector node to which the collector is assigned is obtained in advance based on the collection task information
*
* @param dispatchKey collector task route key: ip+appId
* @return collector node
*/
public Node preDispatchJob(String dispatchKey) {
if (dispatchKey == null || StringUtils.isBlank(dispatchKey)) {
log.error("The dispatch key can not null.");
return null;
}
int dispatchHash = hash(dispatchKey);
return preDispatchJob(dispatchHash);
}
/**
* Obtain the collector node to which the collector is assigned based on the collection task information
*
* @param dispatchHash The task route hash is collected
* @param jobId jobId
* @param isFlushed is has flush this job or wait to dispatch
* @param isFlushed if it has flushed this job or wait to dispatch
* @return collector node
*/
public Node dispatchJob(Integer dispatchHash, Long jobId, boolean isFlushed) {
private CollectorNode dispatchJob(Integer dispatchHash, Long jobId, boolean isFlushed) {
if (dispatchHash == null || hashCircle == null || hashCircle.isEmpty()) {
log.warn("There is no available collector registered. Cache the job {}.", jobId);
dispatchJobCache.add(new DispatchJob(dispatchHash, jobId));
return null;
}
Map.Entry<Integer, Node> ceilEntry = hashCircle.ceilingOrFirstEntry(dispatchHash);
Map.Entry<Integer, CollectorNode> ceilEntry = hashCircle.ceilingOrFirstEntry(dispatchHash);
int virtualKey = ceilEntry.getKey();
Node curNode = ceilEntry.getValue();
CollectorNode curNode = ceilEntry.getValue();
curNode.addJob(virtualKey, dispatchHash, jobId, isFlushed);
return curNode;
@@ -267,25 +313,15 @@ public class ConsistentHash {
* @param dispatchHash The task route hash is collected
* @return collector node
*/
public Node preDispatchJob(Integer dispatchHash) {
private CollectorNode preDispatchJob(Integer dispatchHash) {
if (dispatchHash == null || hashCircle == null || hashCircle.isEmpty()) {
log.warn("There is no available collector registered.");
return null;
}
Map.Entry<Integer, Node> ceilEntry = hashCircle.ceilingOrFirstEntry(dispatchHash);
Map.Entry<Integer, CollectorNode> ceilEntry = hashCircle.ceilingOrFirstEntry(dispatchHash);
return ceilEntry.getValue();
}
/**
* hash long
* @param key long value
* @return hash value
*/
private int hash(long key) {
String keyStr = String.valueOf(key);
return hash(keyStr);
}
/**
* FNV1_32_HASH algorithm
* @param key the key
@@ -314,7 +350,7 @@ public class ConsistentHash {
* dispatch job summary
*/
@AllArgsConstructor
public static class DispatchJob {
private static class DispatchJob {
/**
* dispatch task route key
@@ -327,130 +363,4 @@ public class ConsistentHash {
@Getter
private Long jobId;
}
/**
* collector node machine address
*/
public static class Node {
/**
* collector identity
*/
@Getter
private final String identity;
/**
* collector mode: public or private
*/
private final String mode;
/**
* ip
*/
private final String ip;
/**
* collector On-line time stamp
*/
private final long uptime;
/**
* collector's own performance service quality score 0 - 127
* The number of virtual nodes will be calculated based on this service quality score
*
*/
private final Byte quality;
/**
* use this collector's collect job ID list
* jobId,jobVersion
*/
private AssignJobs assignJobs;
/**
* the collection task ID list mapped by each virtual node corresponding to this node
* Long[] [0]-jobId, [1]-dispatchHash
*/
private Map<Integer, Set<Long[]>> virtualNodeMap;
public Node(String identity, String mode, String ip, long uptime, Byte quality) {
this.identity = identity;
this.mode = mode;
this.ip = ip;
this.uptime = uptime;
this.quality = quality;
assignJobs = new AssignJobs();
virtualNodeMap = new ConcurrentHashMap<>(VIRTUAL_NODE_DEFAULT_SIZE);
}
private synchronized void addJob(Integer virtualNodeKey, Integer dispatchHash, Long jobId, boolean isFlushed) {
if (virtualNodeMap == null) {
virtualNodeMap = new ConcurrentHashMap<>(VIRTUAL_NODE_DEFAULT_SIZE);
}
if (assignJobs == null) {
assignJobs = new AssignJobs();
}
Set<Long[]> virtualNodeJob = virtualNodeMap.computeIfAbsent(virtualNodeKey, k -> ConcurrentHashMap.newKeySet(16));
virtualNodeJob.add(new Long[]{jobId, dispatchHash.longValue()});
if (isFlushed) {
assignJobs.addAssignJob(jobId);
} else {
assignJobs.addAddingJob(jobId);
}
}
/**
* obtain the collection task routed by the specified virtual node according to virtualNodeKey
* @param virtualNodeKey virtualNodeKey
* @return collection task
*/
private Set<Long[]> clearVirtualNodeJobs(Integer virtualNodeKey) {
if (virtualNodeMap == null || virtualNodeMap.isEmpty()) {
return null;
}
Set<Long[]> virtualNodeJobs = virtualNodeMap.remove(virtualNodeKey);
virtualNodeMap.put(virtualNodeKey, ConcurrentHashMap.newKeySet(16));
return virtualNodeJobs;
}
private void addVirtualNodeJobs(Integer virtualHashKey, Set<Long[]> reDispatchJobs) {
if (reDispatchJobs == null) {
return;
}
if (virtualNodeMap == null) {
virtualNodeMap = new ConcurrentHashMap<>(16);
}
virtualNodeMap.computeIfPresent(virtualHashKey, (k, v) -> {
reDispatchJobs.addAll(v);
return v;
});
virtualNodeMap.put(virtualHashKey, reDispatchJobs);
}
public void removeVirtualNodeJob(Long jobId) {
if (jobId == null || virtualNodeMap == null) {
return;
}
for (Set<Long[]> jobSet : virtualNodeMap.values()) {
Optional<Long[]> optional = jobSet.stream().filter(item -> Objects.equals(item[0], jobId)).findFirst();
if (optional.isPresent()) {
jobSet.remove(optional.get());
break;
}
}
}
public AssignJobs getAssignJobs() {
return assignJobs;
}
public void destroy() {
if (assignJobs != null) {
assignJobs.clear();
}
if (virtualNodeMap != null) {
virtualNodeMap.clear();
}
}
}
}
@@ -24,15 +24,8 @@ import org.apache.hertzbeat.common.entity.message.CollectRep;
/**
* Collection job management provides api interface
*/
public interface CollectJobScheduling {
public interface JobOperation {
/**
* Execute a one-time collection task and get the collected data response
* @param job Collect task details
* @return Collection results
*/
List<CollectRep.MetricsData> collectSyncJobData(Job job);
/**
* Execute a one-time collection task and get the collected data response
* @param job Collect task details
@@ -49,13 +42,6 @@ public interface CollectJobScheduling {
*/
long addAsyncCollectJob(Job job, String collector);
/**
* Update the periodic asynchronous collection tasks that have been delivered
* @param modifyJob Collect task details
* @return long Job ID
*/
long updateAsyncCollectJob(Job modifyJob);
/**
* Update the periodic asynchronous collection tasks that have been delivered
* @param modifyJob Collect task details
@@ -52,10 +52,10 @@ import org.springframework.util.StringUtils;
public class SchedulerInit implements CommandLineRunner {
@Autowired
private CollectorScheduling collectorScheduling;
private CollectorOperationReceiver collectorOperationReceiver;
@Autowired
private CollectJobScheduling collectJobScheduling;
private JobOperation jobOperation;
private static final String MAIN_COLLECTOR_NODE_IP = "127.0.0.1";
private static final String DEFAULT_COLLECTOR_VERSION = "DEBUG";
@@ -91,7 +91,7 @@ public class SchedulerInit implements CommandLineRunner {
.ip(MAIN_COLLECTOR_NODE_IP)
.version(DEFAULT_COLLECTOR_VERSION)
.build();
collectorScheduling.collectorGoOnline(CommonConstants.MAIN_COLLECTOR_NODE, collectorInfo);
collectorOperationReceiver.collectorGoOnline(CommonConstants.MAIN_COLLECTOR_NODE, collectorInfo);
// init jobs
List<Monitor> monitors = monitorDao.findMonitorsByStatusNotInAndJobIdNotNull(List.of(CommonConstants.MONITOR_PAUSED_CODE));
List<CollectorMonitorBind> monitorBinds = collectorMonitorBindDao.findAll();
@@ -136,7 +136,7 @@ public class SchedulerInit implements CommandLineRunner {
});
appDefine.setConfigmap(configmaps);
String collector = monitorIdCollectorMap.get(monitor.getId());
long jobId = collectJobScheduling.addAsyncCollectJob(appDefine, collector);
long jobId = jobOperation.addAsyncCollectJob(appDefine, collector);
monitor.setJobId(jobId);
monitorDao.save(monitor);
} catch (Exception e) {
@@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.scheduler.collector;
import org.apache.hertzbeat.common.constants.CollectorStatus;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.manager.pojo.CollectorNode;
import org.apache.hertzbeat.manager.scheduler.AssignJobs;
import java.util.function.BiConsumer;
/**
* Interface for managing collector nodes and their associated jobs.
* Maintains all collector information and provides operations for managing collectors and job assignments.
*/
public interface CollectorKeeper {
/**
* Adds a new collector node to the keeper's management pool.
* @param newNode The collector node to be added to the management system
*/
void addNode(CollectorNode newNode);
/**
* Assigns a monitoring job to a specific collector node.
* @param job The monitoring job to be assigned
* @param collectorId The unique identifier of the target collector node
* @return The collector node that received the job assignment
*/
CollectorNode addJob(Job job, String collectorId);
/**
* Retrieves a collector node by its unique identifier.
* @param collectorId The unique identifier of the collector node
* @return The collector node matching the given ID, or null if not found
*/
CollectorNode getNode(String collectorId);
/**
* Determines the most appropriate collector node for a given job based on scheduling logic.
* @param jobId The unique identifier of the job to be assigned
* @return The collector node selected to handle this job
*/
CollectorNode determineNode(Long jobId);
/**
* Updates the operational status of a collector node.
* @param collectorId The unique identifier of the collector node
* @param collectorStatus The new status to assign to the collector
*/
void changeStatus(String collectorId, CollectorStatus collectorStatus);
/**
* Rebalances job assignments across collector nodes, typically triggered by status changes.
* Uses a callback mechanism to handle job reassignments.
* @param assignJobCollectorConsumer A biconsumer that handles the job reassignment process,
* taking the job assignment logic and collector ID as parameters
*/
void rebalanceJobs(BiConsumer<AssignJobs, String> assignJobCollectorConsumer);
/**
* Removes a job from whichever collector node it is currently assigned to.
* @param jobId The unique identifier of the job to be removed
* @return The collector node from which the job was removed, or null if job wasn't found
*/
CollectorNode removeJob(Long jobId);
}
@@ -28,7 +28,7 @@ import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler;
import org.apache.hertzbeat.common.entity.message.ClusterMsg;
import org.apache.hertzbeat.common.support.CommonThreadPool;
import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler;
import org.apache.hertzbeat.manager.scheduler.SchedulerProperties;
import org.apache.hertzbeat.manager.properties.SchedulerProperties;
import org.apache.hertzbeat.manager.scheduler.netty.process.CollectCyclicDataResponseProcessor;
import org.apache.hertzbeat.manager.scheduler.netty.process.CollectCyclicServiceDiscoveryDataResponseProcessor;
import org.apache.hertzbeat.manager.scheduler.netty.process.CollectOneTimeDataResponseProcessor;
@@ -45,6 +45,8 @@ import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import javax.annotation.PreDestroy;
/**
* manage server
*/
@@ -112,6 +114,7 @@ public class ManageServer implements CommandLineRunner {
}, 10, 3, TimeUnit.SECONDS);
}
@PreDestroy
public void shutdown() {
this.remotingServer.shutdown();
@@ -31,8 +31,9 @@ import org.apache.hertzbeat.common.support.exception.CommonException;
import org.apache.hertzbeat.common.util.IpDomainUtil;
import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.pojo.CollectorNode;
import org.apache.hertzbeat.manager.scheduler.AssignJobs;
import org.apache.hertzbeat.manager.scheduler.ConsistentHash;
import org.apache.hertzbeat.manager.scheduler.ConsistentHashCollectorKeeper;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
import org.apache.hertzbeat.manager.service.CollectorService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -57,7 +58,7 @@ public class CollectorServiceImpl implements CollectorService {
private CollectorMonitorBindDao collectorMonitorBindDao;
@Autowired
private ConsistentHash consistentHash;
private ConsistentHashCollectorKeeper consistentHashCollectorKeeper;
@Autowired(required = false)
private ManageServer manageServer;
@@ -81,7 +82,7 @@ public class CollectorServiceImpl implements CollectorService {
List<CollectorSummary> collectorSummaryList = new LinkedList<>();
for (Collector collector : collectors.getContent()) {
CollectorSummary.CollectorSummaryBuilder summaryBuilder = CollectorSummary.builder().collector(collector);
ConsistentHash.Node node = consistentHash.getNode(collector.getName());
CollectorNode node = consistentHashCollectorKeeper.getNode(collector.getName());
if (node != null && node.getAssignJobs() != null) {
AssignJobs assignJobs = node.getAssignJobs();
summaryBuilder.pinMonitorNum(assignJobs.getPinnedJobs().size());
@@ -60,7 +60,7 @@ import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.pojo.dto.AppCount;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.scheduler.CollectJobScheduling;
import org.apache.hertzbeat.manager.scheduler.JobOperation;
import org.apache.hertzbeat.manager.service.AppService;
import org.apache.hertzbeat.manager.service.ImExportService;
import org.apache.hertzbeat.manager.service.LabelService;
@@ -113,7 +113,7 @@ public class MonitorServiceImpl implements MonitorService {
@Autowired
private AppService appService;
@Autowired
private CollectJobScheduling collectJobScheduling;
private JobOperation jobOperation;
@Autowired
private MonitorDao monitorDao;
@Autowired
@@ -193,11 +193,10 @@ public class MonitorServiceImpl implements MonitorService {
return new Configmap(param.getField(), param.getParamValue(), param.getType());
}).collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
long jobId = collector == null ? collectJobScheduling.addAsyncCollectJob(appDefine, null) :
collectJobScheduling.addAsyncCollectJob(appDefine, collector);
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {}
long jobId = jobOperation.addAsyncCollectJob(appDefine, collector);
detectMonitorSafely(monitor, params, collector);
try {
if (collector != null) {
@@ -217,7 +216,7 @@ public class MonitorServiceImpl implements MonitorService {
paramDao.saveAll(params);
} catch (Exception e) {
log.error("Error while adding monitor: {}", e.getMessage(), e);
collectJobScheduling.cancelAsyncCollectJob(jobId);
jobOperation.cancelAsyncCollectJob(jobId);
throw new MonitorDatabaseException(e.getMessage());
}
}
@@ -510,18 +509,11 @@ public class MonitorServiceImpl implements MonitorService {
List<Configmap> configmaps = params.stream().map(param ->
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
long newJobId;
if (collector == null) {
newJobId = collectJobScheduling.updateAsyncCollectJob(appDefine);
} else {
newJobId = collectJobScheduling.updateAsyncCollectJob(appDefine, collector);
}
long newJobId = jobOperation.updateAsyncCollectJob(appDefine, collector);
monitor.setJobId(newJobId);
// execute only in non paused status
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {}
detectMonitorSafely(monitor, params, collector);
}
// After the update is successfully released, refresh the database
@@ -548,7 +540,7 @@ public class MonitorServiceImpl implements MonitorService {
} catch (Exception e) {
log.error(e.getMessage(), e);
// Repository brushing abnormally cancels the previously delivered task
collectJobScheduling.cancelAsyncCollectJob(monitor.getJobId());
jobOperation.cancelAsyncCollectJob(monitor.getJobId());
throw new MonitorDatabaseException(e.getMessage());
}
}
@@ -578,7 +570,7 @@ public class MonitorServiceImpl implements MonitorService {
for (Monitor monitor : monitors) {
monitorBindDao.deleteByMonitorId(monitor.getId());
collectorMonitorBindDao.deleteCollectorMonitorBindsByMonitorId(monitor.getId());
collectJobScheduling.cancelAsyncCollectJob(monitor.getJobId());
jobOperation.cancelAsyncCollectJob(monitor.getJobId());
applicationContext.publishEvent(new MonitorDeletedEvent(applicationContext, monitor.getId()));
}
}
@@ -692,17 +684,17 @@ public class MonitorServiceImpl implements MonitorService {
// The jobId is not deleted, and the jobId is reused again after the management is started.
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId).collect(Collectors.toSet());
ids.addAll(subMonitorIds);
List<Monitor> managedMonitors = monitorDao.findMonitorsByIdIn(ids)
.stream().filter(monitor ->
monitor.getStatus() != CommonConstants.MONITOR_PAUSED_CODE)
List<Monitor> managedMonitors = monitorDao.findMonitorsByIdIn(ids).stream()
.filter(monitor -> monitor.getStatus() != CommonConstants.MONITOR_PAUSED_CODE)
.peek(monitor -> monitor.setStatus(CommonConstants.MONITOR_PAUSED_CODE))
.collect(Collectors.toList());
if (!CollectionUtils.isEmpty(managedMonitors)) {
for (Monitor monitor : managedMonitors) {
collectJobScheduling.cancelAsyncCollectJob(monitor.getJobId());
}
monitorDao.saveAll(managedMonitors);
if (CollectionUtils.isEmpty(managedMonitors)) {
return;
}
managedMonitors.forEach(monitor -> jobOperation.cancelAsyncCollectJob(monitor.getJobId()));
monitorDao.saveAll(managedMonitors);
}
@Override
@@ -710,9 +702,8 @@ public class MonitorServiceImpl implements MonitorService {
// Update monitoring status Add corresponding monitoring periodic task
Set<Long> subMonitorIds = monitorBindDao.findMonitorBindsByBizIdIn(ids).stream().map(MonitorBind::getMonitorId).collect(Collectors.toSet());
ids.addAll(subMonitorIds);
List<Monitor> unManagedMonitors = monitorDao.findMonitorsByIdIn(ids)
.stream().filter(monitor ->
monitor.getStatus() == CommonConstants.MONITOR_PAUSED_CODE)
List<Monitor> unManagedMonitors = monitorDao.findMonitorsByIdIn(ids).stream()
.filter(monitor -> monitor.getStatus() == CommonConstants.MONITOR_PAUSED_CODE)
.peek(monitor -> monitor.setStatus(CommonConstants.MONITOR_UP_CODE))
.collect(Collectors.toList());
if (unManagedMonitors.isEmpty()) {
@@ -757,13 +748,11 @@ public class MonitorServiceImpl implements MonitorService {
Optional<CollectorMonitorBind> bindOptional =
collectorMonitorBindDao.findCollectorMonitorBindByMonitorId(monitor.getId());
String collector = bindOptional.map(CollectorMonitorBind::getCollector).orElse(null);
long newJobId = collectJobScheduling.addAsyncCollectJob(appDefine, collector);
long newJobId = jobOperation.addAsyncCollectJob(appDefine, collector);
monitor.setJobId(newJobId);
applicationContext.publishEvent(new MonitorDeletedEvent(applicationContext, monitor.getId()));
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {
}
detectMonitorSafely(monitor, params, collector);
}
monitorDao.saveAll(unManagedMonitors);
}
@@ -852,7 +841,7 @@ public class MonitorServiceImpl implements MonitorService {
// if is pinned collector
String collector = monitorIdCollectorMap.get(monitor.getId());
// Delivering a collection task
long newJobId = collectJobScheduling.updateAsyncCollectJob(appDefine, collector);
long newJobId = jobOperation.updateAsyncCollectJob(appDefine, collector);
monitor.setJobId(newJobId);
monitorDao.save(monitor);
} catch (Exception e) {
@@ -932,12 +921,8 @@ public class MonitorServiceImpl implements MonitorService {
new Configmap(param.getField(), param.getParamValue(), param.getType())).collect(Collectors.toList());
appDefine.setConfigmap(configmaps);
appDefine.setSd(true);
List<CollectRep.MetricsData> collectRep;
if (collector != null) {
collectRep = collectJobScheduling.collectSyncJobData(appDefine, collector);
} else {
collectRep = collectJobScheduling.collectSyncJobData(appDefine);
}
List<CollectRep.MetricsData> collectRep = jobOperation.collectSyncJobData(appDefine, collector);
monitor.setStatus(CommonConstants.MONITOR_UP_CODE);
// If the detection result fails, a detection exception is thrown
if (collectRep == null || collectRep.isEmpty()) {
@@ -975,12 +960,7 @@ public class MonitorServiceImpl implements MonitorService {
List<Metrics> availableMetrics = appDefine.getMetrics().stream()
.filter(item -> item.getPriority() == 0).collect(Collectors.toList());
appDefine.setMetrics(availableMetrics);
List<CollectRep.MetricsData> collectRep;
if (collector != null) {
collectRep = collectJobScheduling.collectSyncJobData(appDefine, collector);
} else {
collectRep = collectJobScheduling.collectSyncJobData(appDefine);
}
List<CollectRep.MetricsData> collectRep = jobOperation.collectSyncJobData(appDefine, collector);
monitor.setStatus(CommonConstants.MONITOR_UP_CODE);
// If the detection result fails, a detection exception is thrown
@@ -994,4 +974,10 @@ public class MonitorServiceImpl implements MonitorService {
}
collectRep.forEach(CollectRep.MetricsData::close);
}
private void detectMonitorSafely(Monitor monitor, List<Param> params, String collector) {
try {
detectMonitor(monitor, params, collector);
} catch (Exception ignored) {}
}
}
@@ -19,6 +19,33 @@ spring:
name: ${HOSTNAME:@hertzbeat@}${PID}
profiles:
active: prod
ai:
mcp:
server:
enabled: true
stdio: false
name: sse-mcp-server
version: 1.0.0
resource-change-notification: true
tool-change-notification: true
prompt-change-notification: true
sse-endpoint: /api/sse
sse-message-endpoint: /api/mcp/message
type: SYNC
capabilities:
tool: true
resource: true
prompt: true
completion: true
chat:
client:
enabled: false
openai:
api-key: OPENAI_API_KEY
chat:
options:
model: gpt-4.1-nano-2025-04-14
mvc:
static-path-pattern: /**
jackson:
@@ -38,6 +65,7 @@ spring:
max-file-size: 100MB
max-request-size: 100MB
management:
health:
mail:
@@ -0,0 +1,558 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The monitoring type categoryservice-application service monitoring db-database monitoring custom-custom monitoring os-operating system monitoring
category: bigdata
# The monitoring type eg: linux windows tomcat mysql aws...
app: dolphinscheduler
# The monitoring i18n name
name:
zh-CN: Apache DolphinScheduler
en-US: Apache DolphinScheduler
# The description and help of this monitoring type
help:
zh-CN: Hertzbeat 对 Apache DolphinScheduler (支持v3.3.0或更高版本)通用指标进行测量监控。<br>您可以点击 “<i>新建 Apache DolphinScheduler</i>” 并进行配置,或者选择“<i>更多操作</i>”,导入已有配置。
en-US: Hertzbeat measures and monitors the general metrics of Apache DolphinScheduler (support v3.3.0 or later).<br>You can click on “<i>Create New Apache DolphinScheduler</i>” to configure it, or select “<i>More Actions</i>” to import an existing configuration.
helpLink:
zh-CN: https://hertzbeat.apache.org/zh-cn/docs/help/dolphinscheduler
en-US: https://hertzbeat.apache.org/docs/help/dolphinscheduler
# Input params define for monitoring(render web ui by the definition)
params:
# field-param field key
- field: host
# name-param field display i18n name
name:
zh-CN: 目标Host
en-US: Target Host
# type-param field type(most mapping the html input type)
type: host
# required-true or false
required: true
# field-param field key
- field: port
# name-param field display i18n name
name:
zh-CN: 端口
en-US: Port
# type-param field type(most mapping the html input type)
type: number
# when type is number, range is required
range: '[0,65535]'
# required-true or false
required: true
# default value
defaultValue: 12345
# field-param field key
- field: timeout
# name-param field display i18n name
name:
zh-CN: 查询超时时间
en-US: Query Timeout
# type-param field type(most mapping the html input type)
type: number
# required-true or false
required: false
# hide param-true or false
hide: true
# default value
defaultValue: 6000
# field-param field key
- field: ssl
# name-param field display i18n name
name:
zh-CN: 启用HTTPS
en-US: SSL
# type-param field type(boolean mapping the html h tag)
type: boolean
# required-true or false
required: true
- field: token
# name-param field display i18n name
name:
zh-CN: 令牌
en-US: Token
type: text
limit: 100
required: true
# collect metrics config list
metrics:
- name: master
i18n:
zh-CN: Master
en-US: Master
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 0
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: host
type: 1
i18n:
zh-CN: 主机地址
en-US: Host
- field: port
type: 1
i18n:
zh-CN: 端口
en-US: Port
- field: serverStatus
type: 1
i18n:
zh-CN: 状态
en-US: Server Status
- field: processId
type: 1
i18n:
zh-CN: 进程 ID
en-US: Process Id
- field: runningTime
type: 0
i18n:
zh-CN: 运行时间
en-US: Running Time
- field: cpuUsage
type: 0
unit: '%'
i18n:
zh-CN: 处理器使用量
en-US: CPU Usage
- field: memoryUsage
type: 0
unit: '%'
i18n:
zh-CN: 内存使用量
en-US: Memory Usage
- field: diskUsage
type: 0
unit: '%'
i18n:
zh-CN: 磁盘使用量
en-US: Disk Usage
- field: jvmCpuUsage
type: 0
unit: '%'
i18n:
zh-CN: JVM CPU 使用量
en-US: JVM CPU Usage
- field: jvmMemoryUsage
type: 0
unit: '%'
i18n:
zh-CN: JVM 内存 使用量
en-US: JVM Memory Usage
- field: jvmHeapUsed
type: 0
i18n:
zh-CN: JVM 已使用的堆内存大小
en-US: JVM Heap Used
- field: jvmNonHeapUsed
type: 0
i18n:
zh-CN: JVM 已使用的非堆内存大小
en-US: JVM NonHeap Used
- field: jvmHeapMax
type: 0
i18n:
zh-CN: JVM 配置的最大堆内存大小
en-US: JVM Heap Max
- field: jvmNonHeapMax
type: 0
i18n:
zh-CN: JVM 配置的最大非堆内存大小
en-US: JVM NonHeap Max
# (optional)metrics field alias name, it is used as an alias field to map and convert the collected data and metrics field
aliasFields:
- $.heartBeatInfo
# mapping and conversion expressions, use these and aliasField above to calculate metrics value
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
calculates:
- host=json:apply($.heartBeatInfo).host
- port=json:apply($.heartBeatInfo).port
- serverStatus=json:apply($.heartBeatInfo).serverStatus
- processId=json:apply($.heartBeatInfo).processId
- runningTime=(now()-json:apply($.heartBeatInfo).startupTime)/86400000
- cpuUsage=json:apply($.heartBeatInfo).cpuUsage * 100
- memoryUsage=json:apply($.heartBeatInfo).memoryUsage * 100
- diskUsage=json:apply($.heartBeatInfo).diskUsage * 100
- jvmCpuUsage=json:apply($.heartBeatInfo).jvmCpuUsage
- jvmMemoryUsage=json:apply($.heartBeatInfo).jvmMemoryUsage
- jvmHeapUsed=json:apply($.heartBeatInfo).jvmHeapUsed
- jvmNonHeapUsed=json:apply($.heartBeatInfo).jvmNonHeapUsed
- jvmHeapMax=json:apply($.heartBeatInfo).jvmHeapMax
- jvmNonHeapMax=json:apply($.heartBeatInfo).jvmNonHeapMax
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# the config content when protocol is http
http:
# http host: ipv4 ipv6 domain
host: ^_^host^_^
# http port
port: ^_^port^_^
# http url
url: /dolphinscheduler/monitor/MASTER
# http method: GET POST PUT DELETE PATCH
method: GET
# if enabled https
ssl: ^_^ssl^_^
# http request header content
headers:
token: ^_^token^_^
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: jsonPath
parseScript: '$.data[*]'
- name: worker
i18n:
zh-CN: Worker
en-US: Worker
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 1
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: host
type: 1
i18n:
zh-CN: 主机地址
en-US: Host
- field: port
type: 1
i18n:
zh-CN: 端口
en-US: Port
- field: serverStatus
type: 1
i18n:
zh-CN: 状态
en-US: Server Status
- field: processId
type: 1
i18n:
zh-CN: 进程 ID
en-US: Process Id
- field: runningTime
type: 0
i18n:
zh-CN: 运行时间
en-US: Running Time
- field: cpuUsage
type: 0
unit: '%'
i18n:
zh-CN: 处理器使用量
en-US: CPU Usage
- field: memoryUsage
type: 0
unit: '%'
i18n:
zh-CN: 内存使用量
en-US: Memory Usage
- field: diskUsage
type: 0
unit: '%'
i18n:
zh-CN: 磁盘使用量
en-US: Disk Usage
- field: jvmCpuUsage
type: 0
unit: '%'
i18n:
zh-CN: JVM CPU 使用量
en-US: JVM CPU Usage
- field: jvmMemoryUsage
type: 0
unit: '%'
i18n:
zh-CN: JVM 内存 使用量
en-US: JVM Memory Usage
- field: jvmHeapUsed
type: 0
i18n:
zh-CN: JVM 已使用的堆内存大小
en-US: JVM Heap Used
- field: jvmNonHeapUsed
type: 0
i18n:
zh-CN: JVM 已使用的非堆内存大小
en-US: JVM NonHeap Used
- field: jvmHeapMax
type: 0
i18n:
zh-CN: JVM 配置的最大堆内存大小
en-US: JVM Heap Max
- field: jvmNonHeapMax
type: 0
i18n:
zh-CN: JVM 配置的最大非堆内存大小
en-US: JVM NonHeap Max
- field: workerHostWeight
type: 0
i18n:
zh-CN: 权重
en-US: Weight
- field: threadPoolUsage
type: 0
unit: '%'
i18n:
zh-CN: 线程池使用量
en-US: Thread Pool Usage
- field: workerGroup
type: 1
i18n:
zh-CN: Worker 组
en-US: Worker Group
# (optional)metrics field alias name, it is used as an alias field to map and convert the collected data and metrics field
aliasFields:
- $.heartBeatInfo
# mapping and conversion expressions, use these and aliasField above to calculate metrics value
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
calculates:
- host=json:apply($.heartBeatInfo).host
- port=json:apply($.heartBeatInfo).port
- serverStatus=json:apply($.heartBeatInfo).serverStatus
- processId=json:apply($.heartBeatInfo).processId
- runningTime=(now()-json:apply($.heartBeatInfo).startupTime)/86400000
- cpuUsage=json:apply($.heartBeatInfo).cpuUsage * 100
- memoryUsage=json:apply($.heartBeatInfo).memoryUsage * 100
- diskUsage=json:apply($.heartBeatInfo).diskUsage * 100
- jvmCpuUsage=json:apply($.heartBeatInfo).jvmCpuUsage
- jvmMemoryUsage=json:apply($.heartBeatInfo).jvmMemoryUsage
- jvmHeapUsed=json:apply($.heartBeatInfo).jvmHeapUsed
- jvmNonHeapUsed=json:apply($.heartBeatInfo).jvmNonHeapUsed
- jvmHeapMax=json:apply($.heartBeatInfo).jvmHeapMax
- jvmNonHeapMax=json:apply($.heartBeatInfo).jvmNonHeapMax
- workerHostWeight=json:apply($.heartBeatInfo).workerHostWeight
- threadPoolUsage=json:apply($.heartBeatInfo).threadPoolUsage
- workerGroup=json:apply($.heartBeatInfo).workerGroup
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# the config content when protocol is http
http:
# http host: ipv4 ipv6 domain
host: ^_^host^_^
# http port
port: ^_^port^_^
# http url
url: /dolphinscheduler/monitor/WORKER
# http method: GET POST PUT DELETE PATCH
method: GET
# if enabled https
ssl: ^_^ssl^_^
# http request header content
headers:
token: ^_^token^_^
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: jsonPath
parseScript: '$.data[*]'
- name: alert-server
i18n:
zh-CN: Alert Server
en-US: Alert Server
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 0
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: host
type: 1
i18n:
zh-CN: 主机地址
en-US: Host
- field: port
type: 1
i18n:
zh-CN: 端口
en-US: Port
- field: serverStatus
type: 1
i18n:
zh-CN: 状态
en-US: Server Status
- field: processId
type: 1
i18n:
zh-CN: 进程 ID
en-US: Process Id
- field: runningTime
type: 0
i18n:
zh-CN: 运行时间
en-US: Running Time
- field: cpuUsage
type: 0
unit: '%'
i18n:
zh-CN: 处理器使用量
en-US: CPU Usage
- field: memoryUsage
type: 0
unit: '%'
i18n:
zh-CN: 内存使用量
en-US: Memory Usage
- field: diskUsage
type: 0
unit: '%'
i18n:
zh-CN: 磁盘使用量
en-US: Disk Usage
- field: jvmCpuUsage
type: 0
unit: '%'
i18n:
zh-CN: JVM CPU 使用量
en-US: JVM CPU Usage
- field: jvmMemoryUsage
type: 0
unit: '%'
i18n:
zh-CN: JVM 内存 使用量
en-US: JVM Memory Usage
- field: jvmHeapUsed
type: 0
i18n:
zh-CN: JVM 已使用的堆内存大小
en-US: JVM Heap Used
- field: jvmNonHeapUsed
type: 0
i18n:
zh-CN: JVM 已使用的非堆内存大小
en-US: JVM NonHeap Used
- field: jvmHeapMax
type: 0
i18n:
zh-CN: JVM 配置的最大堆内存大小
en-US: JVM Heap Max
- field: jvmNonHeapMax
type: 0
i18n:
zh-CN: JVM 配置的最大非堆内存大小
en-US: JVM NonHeap Max
# (optional)metrics field alias name, it is used as an alias field to map and convert the collected data and metrics field
aliasFields:
- $.heartBeatInfo
# mapping and conversion expressions, use these and aliasField above to calculate metrics value
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
calculates:
- host=json:apply($.heartBeatInfo).host
- port=json:apply($.heartBeatInfo).port
- serverStatus=json:apply($.heartBeatInfo).serverStatus
- processId=json:apply($.heartBeatInfo).processId
- runningTime=(now()-json:apply($.heartBeatInfo).startupTime)/86400000
- cpuUsage=json:apply($.heartBeatInfo).cpuUsage * 100
- memoryUsage=json:apply($.heartBeatInfo).memoryUsage * 100
- diskUsage=json:apply($.heartBeatInfo).diskUsage * 100
- jvmCpuUsage=json:apply($.heartBeatInfo).jvmCpuUsage
- jvmMemoryUsage=json:apply($.heartBeatInfo).jvmMemoryUsage
- jvmHeapUsed=json:apply($.heartBeatInfo).jvmHeapUsed
- jvmNonHeapUsed=json:apply($.heartBeatInfo).jvmNonHeapUsed
- jvmHeapMax=json:apply($.heartBeatInfo).jvmHeapMax
- jvmNonHeapMax=json:apply($.heartBeatInfo).jvmNonHeapMax
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# the config content when protocol is http
http:
# http host: ipv4 ipv6 domain
host: ^_^host^_^
# http port
port: ^_^port^_^
# http url
url: /dolphinscheduler/monitor/ALERT_SERVER
# http method: GET POST PUT DELETE PATCH
method: GET
# if enabled https
ssl: ^_^ssl^_^
# http request header content
headers:
token: ^_^token^_^
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: jsonPath
parseScript: '$.data[*]'
- name: database
i18n:
zh-CN: 数据库
en-US: Database
# metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
# priority 0's metrics is availability metrics, it will be scheduled first, only availability metrics collect success will the scheduling continue
priority: 2
# collect metrics content
fields:
# field-metric name, type-metric type(0-number,1-string), unit-metric unit('%','ms','MB'), label-whether it is a metrics label field
- field: dbType
type: 1
i18n:
zh-CN: 数据库类型
en-US: Database Type
- field: state
type: 1
i18n:
zh-CN: 状态
en-US: State
- field: maxConnections
type: 0
i18n:
zh-CN: 最大连接数
en-US: Max Connections
- field: threadsConnections
type: 0
i18n:
zh-CN: 当前连接数
en-US: Threads Connections
- field: threadsRunningConnections
type: 0
i18n:
zh-CN: 当前活跃连接数
en-US: Threads Running Connections
# (optional)metrics field alias name, it is used as an alias field to map and convert the collected data and metrics field
aliasFields:
- dbType
- state
- maxConnections
- threadsConnections
- threadsRunningConnections
# mapping and conversion expressions, use these and aliasField above to calculate metrics value
# eg: cores=core1+core2, usage=usage, waitTime=allTime-runningTime
calculates:
- dbType=dbType
- state=state
- maxConnections=maxConnections
- threadsConnections=threadsConnections
- threadsRunningConnections=threadsRunningConnections
# the protocol used for monitoring, eg: sql, ssh, http, telnet, wmi, snmp, sdk
protocol: http
# the config content when protocol is http
http:
# http host: ipv4 ipv6 domain
host: ^_^host^_^
# http port
port: ^_^port^_^
# http url
url: /dolphinscheduler/monitor/databases
# http method: GET POST PUT DELETE PATCH
method: GET
# if enabled https
ssl: ^_^ssl^_^
# http request header content
headers:
token: ^_^token^_^
# http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring, prometheus-prometheus exporter rule
parseType: jsonPath
parseScript: '$.data[0]'
@@ -185,7 +185,7 @@ metrics:
# SQL Query MethodoneRow, multiRow, columns
queryType: oneRow
# sql
sql: select * from sys.v_$instance
sql: "select * from sys.v_$instance"
# JDBC url
url: ^_^url^_^
@@ -246,7 +246,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: multiRow
sql: select file_id, file_name, tablespace_name, status, bytes / 1024 / 1024 as bytes, blocks from dba_data_files
sql: "select file_id, file_name, tablespace_name, status, bytes / 1024 / 1024 as bytes, blocks from dba_data_files"
url: ^_^url^_^
- name: total_sessions
@@ -273,7 +273,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: oneRow
sql: select count(*) as count from v$session
sql: "select count(*) as count from v$session"
url: ^_^url^_^
- name: active_sessions
@@ -300,7 +300,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: oneRow
sql: select count(*) as count from v$session where username is not null and status = 'ACTIVE'
sql: "select count(*) as count from v$session where username is not null and status = 'ACTIVE'"
url: ^_^url^_^
- name: background_sessions
@@ -327,7 +327,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: oneRow
sql: select count(*) as count from v$session where username is null
sql: "select count(*) as count from v$session where username is null"
url: ^_^url^_^
- name: connection
@@ -361,7 +361,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: oneRow
sql: SELECT username, count( username ) as count FROM v$session WHERE username IS NOT NULL GROUP BY username
sql: "SELECT username, count( username ) as count FROM v$session WHERE username IS NOT NULL GROUP BY username"
url: ^_^url^_^
- name: performance
@@ -411,13 +411,13 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: columns
sql: select metric_name, value from gv$sysmetric where metric_name = 'I/O Megabytes per Second' or metric_name = 'User Transaction Per Sec' or metric_name = 'I/O Requests per Second'
sql: "select metric_name, value from gv$sysmetric where metric_name = 'I/O Megabytes per Second' or metric_name = 'User Transaction Per Sec' or metric_name = 'I/O Requests per Second'"
url: ^_^url^_^
- name: percentage
i18n:
zh-CN: 百分比
zh-CN: 表空间百分比
en-US: Percentage
ja-JP: パーセント
priority: 7
@@ -499,7 +499,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: oneRow
sql: select count(*) as process_count from v$process
sql: "select count(*) as process_count from v$process"
url: ^_^url^_^
- name: transaction
@@ -540,7 +540,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: columns
sql: select metric_name, value from gv$sysmetric where metric_name = 'User Commits Per Sec' or metric_name = 'User Rollbacks Per Sec'
sql: "select metric_name, value from gv$sysmetric where metric_name = 'User Commits Per Sec' or metric_name = 'User Rollbacks Per Sec'"
url: ^_^url^_^
- name: wait
@@ -635,7 +635,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: columns
sql: select wait_class, sum(time_waited) total_wait_time from v$active_session_history where session_state = 'WAITING' GROUP BY wait_class ORDER BY total_wait_time DESC
sql: "select wait_class, sum(time_waited) total_wait_time from v$active_session_history where session_state = 'WAITING' GROUP BY wait_class ORDER BY total_wait_time DESC"
url: ^_^url^_^
- name: cpu_stats
@@ -669,7 +669,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: multiRow
sql: select stat_name as type, value as num from v$osstat where stat_name like '%CPU%' or stat_name like '%TIME'
sql: "select stat_name as type, value as num from v$osstat where stat_name like '%CPU%' or stat_name like '%TIME'"
url: ^_^url^_^
- name: mem_stats
@@ -703,7 +703,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: multiRow
sql: select stat_name as type, value as num from v$osstat where stat_name like '%BYTES'
sql: "select stat_name as type, value as num from v$osstat where stat_name like '%BYTES'"
url: ^_^url^_^
- name: cache_hit_ratio
@@ -742,7 +742,7 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: columns
sql: select metric_name, value from gv$sysmetric where metric_name like '%Cache Hit Ratio' order by end_time asc
sql: "select metric_name, value from gv$sysmetric where metric_name like '%Cache Hit Ratio' order by end_time asc"
url: ^_^url^_^
- name: slow_query
@@ -832,5 +832,69 @@ metrics:
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: multiRow
sql: SELECT * FROM (SELECT sql_id, child_number, executions, ROUND(CASE WHEN executions = 0 THEN NULL ELSE elapsed_time / (executions*1000000) END,4) AS per_secs, cpu_time / 1000000 AS cpu_secs, buffer_gets, disk_reads, fetches, parse_calls, optimizer_cost, sql_text FROM v$sql ) where rownum <= 10 ORDER BY per_secs DESC
sql: "SELECT * FROM (SELECT sql_id, child_number, executions, ROUND(CASE WHEN executions = 0 THEN NULL ELSE elapsed_time / (executions*1000000) END,4) AS per_secs, cpu_time / 1000000 AS cpu_secs, buffer_gets, disk_reads, fetches, parse_calls, optimizer_cost, sql_text FROM v$sql ) where rownum <= 10 ORDER BY per_secs DESC"
url: ^_^url^_^
- name: users
i18n:
zh-CN: 用户信息
en-US: Users
ja-JP: スロークエリ
priority: 15
fields:
- field: username
type: 1
i18n:
zh-CN: 用户名
en-US: User Name
ja-JP: ユーザー名
- field: account_status
type: 1
i18n:
zh-CN: 账号状态
en-US: Account Status
ja-JP: アカウントのステータス
- field: lock_date
type: 1
i18n:
zh-CN: 锁定时间
en-US: Lock Date
ja-JP: ロック日付
- field: expiry_date
type: 1
i18n:
zh-CN: 密码失效时间
en-US: Password Expiry Date
ja-JP: パスワードの有効期限
- field: expiry_seconds
type: 0
i18n:
zh-CN: 密码剩余有效时间
en-US: Password Validity Period Remaining
ja-JP: パスワードの有効期間残り
unit:
- field: created
type: 1
i18n:
zh-CN: 创建时间
en-US: Creation Date
ja-JP: 作成日
- field: authentication_type
type: 1
i18n:
zh-CN: 认证类型
en-US: Authentication Type
ja-JP: 認証タイプ
protocol: jdbc
jdbc:
host: ^_^host^_^
port: ^_^port^_^
platform: oracle
username: ^_^username^_^
password: ^_^password^_^
database: ^_^database^_^
timeout: ^_^timeout^_^
reuseConnection: ^_^reuseConnection^_^
queryType: multiRow
sql: "SELECT username,account_status,lock_date,expiry_date,ceil((expiry_date-sysdate)* 24 * 60 * 60) as expiry_seconds,created,authentication_type FROM DBA_USERS ORDER BY expiry_date ASC"
url: ^_^url^_^
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -66,7 +66,8 @@ resourceRole:
- /api/bulletin/**===post===[admin,user]
- /api/bulletin/**===put===[admin,user]
- /api/bulletin/**===delete===[admin]
- /api/sse/**===get===[admin,user]
- /api/sse/**===post===[admin,user]
# config the resource restful api that need bypass auth protection
# rule: api===method
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
@@ -29,6 +29,7 @@ import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.pojo.CollectorNode;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
import org.apache.hertzbeat.manager.service.AppService;
import org.junit.jupiter.api.BeforeEach;
@@ -63,7 +64,7 @@ public class CollectorJobSchedulerTest {
@InjectMocks
private CollectorJobScheduler collectorJobScheduler;
@Mock
private ConsistentHash consistentHash;
private ConsistentHashCollectorKeeper consistentHashCollectorKeeper;
@Mock
private CollectorDao collectorDao;
@Mock
@@ -85,8 +86,8 @@ public class CollectorJobSchedulerTest {
public void testCollectSyncJobData() {
assertDoesNotThrow(() -> {
Job job = new Job();
when(consistentHash.preDispatchJob(any(String.class))).thenReturn(null);
List<?> list = collectorJobScheduler.collectSyncJobData(job);
when(consistentHashCollectorKeeper.determineNode(any(Long.class))).thenReturn(null);
List<?> list = collectorJobScheduler.collectSyncJobData(job, null);
assertEquals(1, list.size());
});
}
@@ -127,9 +128,8 @@ public class CollectorJobSchedulerTest {
appDefine.setParams(Collections.emptyList());
when(appService.getAppDefine(anyString())).thenReturn(appDefine);
ConsistentHash.Node node = new ConsistentHash.Node(identity, collector.getMode(),
collector.getIp(), System.currentTimeMillis(), null);
when(consistentHash.getNode("collector-1")).thenReturn(node);
CollectorNode node = new CollectorNode(identity, collector.getMode(), collector.getIp(), System.currentTimeMillis(), null);
when(consistentHashCollectorKeeper.addJob(appDefine, identity)).thenReturn(node);
ManageServer manageServer = mock(ManageServer.class);
collectorJobScheduler.setManageServer(manageServer);
@@ -138,7 +138,7 @@ public class CollectorJobSchedulerTest {
// Capture the parameters of sendMsg
ArgumentCaptor<ClusterMsg.Message> msgCaptor = ArgumentCaptor.forClass(ClusterMsg.Message.class);
verify(manageServer, atLeastOnce()).sendMsg(eq("collector-1"), msgCaptor.capture());
verify(manageServer, atLeastOnce()).sendMsg(eq(identity), msgCaptor.capture());
ClusterMsg.Message message = msgCaptor.getValue();
Job job = JsonUtil.fromJson(message.getMsg().toStringUtf8(), Job.class);
@@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.scheduler;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.hertzbeat.common.constants.CollectorStatus;
import org.apache.hertzbeat.common.entity.job.Job;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
import org.apache.hertzbeat.manager.pojo.CollectorNode;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/**
* Test case for {@link ConsistentHashCollectorKeeper}
*/
public class ConsistentHashCollectorKeeperTest {
private ConsistentHashCollectorKeeper consistentHashCollectorKeeper;
@BeforeEach
void setUp() {
consistentHashCollectorKeeper = new ConsistentHashCollectorKeeper();
}
@Test
void testAddNode() {
long jobId1 = SnowFlakeIdGenerator.generateId();
long jobId2 = SnowFlakeIdGenerator.generateId();
long jobId3 = SnowFlakeIdGenerator.generateId();
CollectorNode node1 = new CollectorNode("node1", "public", "192.168.0.1", System.currentTimeMillis(), (byte) 10);
CollectorNode node2 = new CollectorNode("node2", "public", "192.168.0.2", System.currentTimeMillis(), (byte) 10);
consistentHashCollectorKeeper.addNode(node1);
consistentHashCollectorKeeper.determineNode(jobId1);
consistentHashCollectorKeeper.determineNode(jobId2);
consistentHashCollectorKeeper.determineNode(jobId3);
consistentHashCollectorKeeper.addNode(node2);
assertTrue(node2.getAssignJobs().getAddingJobs().containsAll(node1.getAssignJobs().getRemovingJobs()));
assertTrue(node2.getAssignJobs().getAddingJobs().containsAll(node2.getAssignJobs().getRemovingJobs()));
assertSame(consistentHashCollectorKeeper.getNode("node1"), node1);
assertSame(consistentHashCollectorKeeper.getNode("node2"), node2);
}
@Test
void testDispatchJob() {
long jobId1 = SnowFlakeIdGenerator.generateId();
CollectorNode res1 = consistentHashCollectorKeeper.determineNode(jobId1);
assertNull(res1);
CollectorNode node1 = new CollectorNode("node1", "public", "192.168.0.1", System.currentTimeMillis(), (byte) 10);
consistentHashCollectorKeeper.addNode(node1);
long jobId2 = SnowFlakeIdGenerator.generateId();
CollectorNode res2 = consistentHashCollectorKeeper.determineNode(jobId2);
assertSame(res2, node1);
assertTrue(consistentHashCollectorKeeper.getDispatchJobCache().isEmpty());
}
@Test
void testRemoveNode() {
CollectorNode node1 = new CollectorNode("node1", "public", "192.168.0.1", System.currentTimeMillis(), (byte) 10);
CollectorNode node2 = new CollectorNode("node2", "public", "192.168.0.2", System.currentTimeMillis(), (byte) 10);
consistentHashCollectorKeeper.addNode(node1);
consistentHashCollectorKeeper.addNode(node2);
long jobId1 = SnowFlakeIdGenerator.generateId();
long jobId2 = SnowFlakeIdGenerator.generateId();
long jobId3 = SnowFlakeIdGenerator.generateId();
long jobId4 = SnowFlakeIdGenerator.generateId();
consistentHashCollectorKeeper.addJob(Job.builder().id(jobId1).monitorId(jobId1).build(), null);
consistentHashCollectorKeeper.addJob(Job.builder().id(jobId2).monitorId(jobId2).build(), null);
consistentHashCollectorKeeper.addJob(Job.builder().id(jobId3).monitorId(jobId3).build(), null);
consistentHashCollectorKeeper.addJob(Job.builder().id(jobId4).monitorId(jobId4).build(), null);
consistentHashCollectorKeeper.changeStatus(node2.getIdentity(), CollectorStatus.OFFLINE);
assertTrue(node1.getAssignJobs().getAddingJobs().containsAll(node2.getAssignJobs().getRemovingJobs()));
assertSame(consistentHashCollectorKeeper.getNode("node1"), node1);
assertNull(consistentHashCollectorKeeper.getNode("node2"));
consistentHashCollectorKeeper.changeStatus(node1.getIdentity(), CollectorStatus.OFFLINE);
assertEquals(4, consistentHashCollectorKeeper.getDispatchJobCache().size());
}
}
@@ -1,100 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.scheduler;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/**
* Test case for {@link ConsistentHash}
*/
public class ConsistentHashTest {
private ConsistentHash consistentHash;
@BeforeEach
void setUp() {
consistentHash = new ConsistentHash();
}
@Test
void testAddNode() {
String job1 = "job1";
long jobId1 = SnowFlakeIdGenerator.generateId();
String job2 = "job2";
long jobId2 = SnowFlakeIdGenerator.generateId();
String job3 = "job3";
long jobId3 = SnowFlakeIdGenerator.generateId();
ConsistentHash.Node node1 = new ConsistentHash.Node("node1", "public", "192.168.0.1", System.currentTimeMillis(), (byte) 10);
ConsistentHash.Node node2 = new ConsistentHash.Node("node2", "public", "192.168.0.2", System.currentTimeMillis(), (byte) 10);
consistentHash.addNode(node1);
consistentHash.dispatchJob(job1, jobId1);
consistentHash.dispatchJob(job2, jobId2);
consistentHash.dispatchJob(job3, jobId3);
consistentHash.addNode(node2);
assertTrue(node2.getAssignJobs().getAddingJobs().containsAll(node1.getAssignJobs().getRemovingJobs()));
assertTrue(node2.getAssignJobs().getAddingJobs().containsAll(node2.getAssignJobs().getRemovingJobs()));
assertSame(consistentHash.getNode("node1"), node1);
assertSame(consistentHash.getNode("node2"), node2);
}
@Test
void testDispatchJob() {
String job1 = "job1";
long jobId1 = SnowFlakeIdGenerator.generateId();
ConsistentHash.Node res1 = consistentHash.dispatchJob(job1, jobId1);
assertNull(res1);
ConsistentHash.Node node1 = new ConsistentHash.Node("node1", "public", "192.168.0.1", System.currentTimeMillis(), (byte) 10);
consistentHash.addNode(node1);
String job2 = "job2";
long jobId2 = SnowFlakeIdGenerator.generateId();
ConsistentHash.Node res2 = consistentHash.dispatchJob(job2, jobId2);
assertSame(res2, node1);
assertTrue(consistentHash.getDispatchJobCache().isEmpty());
}
@Test
void testRemoveNode() {
String job1 = "job1";
long jobId1 = SnowFlakeIdGenerator.generateId();
String job2 = "job2";
long jobId2 = SnowFlakeIdGenerator.generateId();
String job3 = "job3";
long jobId3 = SnowFlakeIdGenerator.generateId();
ConsistentHash.Node node1 = new ConsistentHash.Node("node1", "public", "192.168.0.1", System.currentTimeMillis(), (byte) 10);
ConsistentHash.Node node2 = new ConsistentHash.Node("node2", "public", "192.168.0.2", System.currentTimeMillis(), (byte) 10);
consistentHash.addNode(node1);
consistentHash.addNode(node2);
consistentHash.dispatchJob(job1, jobId1);
consistentHash.dispatchJob(job2, jobId2);
consistentHash.dispatchJob(job3, jobId3);
consistentHash.removeNode(node2.getIdentity());
assertTrue(node1.getAssignJobs().getAddingJobs().containsAll(node2.getAssignJobs().getRemovingJobs()));
assertSame(consistentHash.getNode("node1"), node1);
assertNull(consistentHash.getNode("node2"));
consistentHash.removeNode(node1.getIdentity());
assertEquals(3, consistentHash.getDispatchJobCache().size());
}
}
@@ -32,7 +32,7 @@ import org.apache.hertzbeat.common.entity.manager.Collector;
import org.apache.hertzbeat.common.support.exception.CommonException;
import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.scheduler.ConsistentHash;
import org.apache.hertzbeat.manager.scheduler.ConsistentHashCollectorKeeper;
import org.apache.hertzbeat.manager.scheduler.netty.ManageServer;
import org.apache.hertzbeat.manager.service.impl.CollectorServiceImpl;
import org.junit.jupiter.api.Test;
@@ -60,7 +60,7 @@ public class CollectorServiceTest {
private CollectorDao collectorDao;
@Mock
private ConsistentHash consistentHash;
private ConsistentHashCollectorKeeper consistentHashCollectorKeeper;
@Mock
private CollectorMonitorBindDao collectorMonitorBindDao;
@@ -46,7 +46,7 @@ import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.pojo.dto.AppCount;
import org.apache.hertzbeat.manager.pojo.dto.MonitorDto;
import org.apache.hertzbeat.manager.scheduler.CollectJobScheduling;
import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler;
import org.apache.hertzbeat.manager.service.impl.MonitorServiceImpl;
import org.apache.hertzbeat.manager.support.exception.MonitorDatabaseException;
import org.apache.hertzbeat.manager.support.exception.MonitorDetectException;
@@ -105,7 +105,7 @@ class MonitorServiceTest {
private LabelService tagService;
@Mock
private CollectJobScheduling collectJobScheduling;
private CollectorJobScheduler jobOperation;
@Mock
private AlertDefineBindDao alertDefineBindDao;
@@ -143,7 +143,7 @@ class MonitorServiceTest {
when(appService.getAppDefine(monitor.getApp())).thenReturn(job);
List<CollectRep.MetricsData> collectRep = new ArrayList<>();
when(collectJobScheduling.collectSyncJobData(job)).thenReturn(collectRep);
when(jobOperation.collectSyncJobData(job, null)).thenReturn(collectRep);
List<Param> params = Collections.singletonList(new Param());
assertThrows(MonitorDetectException.class, () -> monitorService.detectMonitor(monitor, params, null));
@@ -170,7 +170,7 @@ class MonitorServiceTest {
CollectRep.MetricsData failCode = CollectRep.MetricsData.newBuilder()
.setCode(CollectRep.Code.TIMEOUT).setMsg("collect timeout").build();
collectRep.add(failCode);
when(collectJobScheduling.collectSyncJobData(job)).thenReturn(collectRep);
when(jobOperation.collectSyncJobData(job, null)).thenReturn(collectRep);
List<Param> params = Collections.singletonList(new Param());
assertThrows(MonitorDetectException.class, () -> monitorService.detectMonitor(monitor, params, null));
@@ -186,7 +186,7 @@ class MonitorServiceTest {
.build();
Job job = new Job();
when(appService.getAppDefine(monitor.getApp())).thenReturn(job);
when(collectJobScheduling.addAsyncCollectJob(job, null)).thenReturn(1L);
when(jobOperation.addAsyncCollectJob(job, null)).thenReturn(1L);
when(monitorDao.save(monitor)).thenReturn(monitor);
List<Param> params = Collections.singletonList(new Param());
when(paramDao.saveAll(params)).thenReturn(params);
@@ -203,7 +203,7 @@ class MonitorServiceTest {
.build();
Job job = new Job();
when(appService.getAppDefine(monitor.getApp())).thenReturn(job);
when(collectJobScheduling.addAsyncCollectJob(job, null)).thenReturn(1L);
when(jobOperation.addAsyncCollectJob(job, null)).thenReturn(1L);
List<Param> params = Collections.singletonList(new Param());
when(monitorDao.save(monitor)).thenThrow(RuntimeException.class);
assertThrows(MonitorDatabaseException.class, () -> monitorService.addMonitor(monitor, params, null, null));
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.GZIPOutputStream;
import com.google.common.collect.Maps;
@@ -106,8 +107,8 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage {
private final BlockingQueue<VictoriaMetricsDataStorage.VictoriaMetricsContent> metricsBufferQueue;
private HashedWheelTimer metricsFlushTimer = null;
private MetricsFlushTask metricsFlushtask = null;
private final VictoriaMetricsProperties.InsertConfig insertConfig;
private final AtomicBoolean draining = new AtomicBoolean(false);
public VictoriaMetricsDataStorage(VictoriaMetricsProperties victoriaMetricsProperties, RestTemplate restTemplate) {
if (victoriaMetricsProperties == null) {
@@ -129,8 +130,8 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage {
thread.setDaemon(true);
return thread;
}, 1, TimeUnit.SECONDS, 512);
metricsFlushtask = new MetricsFlushTask();
this.metricsFlushTimer.newTimeout(metricsFlushtask, 0, TimeUnit.SECONDS);
// start flush interval timer
this.metricsFlushTimer.newTimeout(new MetricsFlushTask(null), insertConfig.flushInterval(), TimeUnit.SECONDS);
}
private boolean checkVictoriaMetricsDatasourceAvailable() {
@@ -591,36 +592,63 @@ public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage {
}
}
// Refresh in advance to avoid waiting
if (metricsBufferQueue.size() >= insertConfig.bufferSize() * 0.8) {
if (metricsBufferQueue.size() >= insertConfig.bufferSize() * 0.8
&& draining.compareAndSet(false, true)) {
triggerImmediateFlush();
}
}
private void triggerImmediateFlush() {
metricsFlushTimer.newTimeout(metricsFlushtask, 0, TimeUnit.MILLISECONDS);
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch = new ArrayList<>(insertConfig.bufferSize());
metricsBufferQueue.drainTo(batch, insertConfig.bufferSize());
draining.set(false);
if (!batch.isEmpty()) {
metricsFlushTimer.newTimeout(new MetricsFlushTask(batch), 0, TimeUnit.MILLISECONDS);
}
}
/**
* Regularly refresh the buffer queue to the vm
*/
private class MetricsFlushTask implements TimerTask {
private final List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch;
public MetricsFlushTask(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch) {
this.batch = batch;
}
@Override
public void run(Timeout timeout) {
try {
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch = new ArrayList<>(insertConfig.bufferSize());
metricsBufferQueue.drainTo(batch, insertConfig.bufferSize());
if (!batch.isEmpty()) {
doSaveData(batch);
log.debug("[Victoria Metrics] Flushed {} metrics items", batch.size());
}
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
metricsFlushTimer.newTimeout(this, insertConfig.flushInterval(), TimeUnit.SECONDS);
log.debug("[Victoria Metrics] Rescheduled next flush task in {} seconds.", insertConfig.flushInterval());
if (batch == null) {
// If the batch is null, it means that the timer is triggered by flush interval timer
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batchT = new ArrayList<>(insertConfig.bufferSize());
metricsBufferQueue.drainTo(batchT, insertConfig.bufferSize());
triggerDoSaveData(batchT);
// Reschedule the next flush task
triggerIntervalFlushTimer();
} else {
// If the batch is not null, it means that the timer is triggered by the immediate flush
triggerDoSaveData(batch);
}
} catch (Exception e) {
log.error("[VictoriaMetrics] flush task error: {}", e.getMessage(), e);
}
}
private void triggerDoSaveData(List<VictoriaMetricsContent> batch) {
if (!batch.isEmpty()) {
doSaveData(batch);
log.debug("[Victoria Metrics] Flushed {} metrics items", batch.size());
}
}
private void triggerIntervalFlushTimer() {
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
metricsFlushTimer.newTimeout(new MetricsFlushTask(null), insertConfig.flushInterval(), TimeUnit.SECONDS);
log.debug("[Victoria Metrics] Rescheduled next flush task in {} seconds.", insertConfig.flushInterval());
}
}
}
/**
@@ -21,9 +21,8 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.startsWith;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.times;
import static org.assertj.core.api.Assertions.assertThat;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
@@ -53,6 +52,7 @@ import org.springframework.web.client.RestTemplate;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Test case for {@link VictoriaMetricsDataStorage}
@@ -72,18 +72,33 @@ class VictoriaMetricsDataStorageTest {
private VictoriaMetricsDataStorage victoriaMetricsDataStorage;
private final AtomicInteger postForEntityCount = new AtomicInteger(0);
@BeforeEach
void setUp() {
when(victoriaMetricsProperties.enabled()).thenReturn(true);
when(victoriaMetricsProperties.url()).thenReturn("http://localhost:8428");
when(victoriaMetricsProperties.username()).thenReturn("root");
when(victoriaMetricsProperties.password()).thenReturn("root");
// on successful write, VictoriaMetrics returns HTTP 204 (No Content)
when(responseEntity.getStatusCode()).thenReturn(HttpStatus.NO_CONTENT);
when(restTemplate.exchange(anyString(), eq(HttpMethod.GET), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
when(restTemplate.postForEntity(anyString(), any(HttpEntity.class), eq(String.class)))
.thenReturn(responseEntity);
when(restTemplate.exchange(
anyString(),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(String.class)
)).thenReturn(responseEntity);
when(restTemplate.postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(HttpEntity.class),
eq(String.class)
)).thenAnswer(invocation -> {
postForEntityCount.incrementAndGet();
return responseEntity;
});
}
@Test
@@ -92,14 +107,11 @@ class VictoriaMetricsDataStorageTest {
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
// execute one-time data insertion
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the timer's first insertion task execution and verify if it was called once
Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() ->
verify(restTemplate, times(1)).postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(HttpEntity.class),
eq(String.class)
)
);
// wait for the timer's first insertion task execution and verify if it was called once (default 3 seconds)
Awaitility.await()
.pollInterval(2, TimeUnit.SECONDS)
.atMost(7, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
}
@Test
@@ -109,28 +121,15 @@ class VictoriaMetricsDataStorageTest {
10, Integer.MAX_VALUE, new VictoriaMetricsProperties.Compression(false)));
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the timer to execute its first insertion task
Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() ->
verify(restTemplate, times(1)).postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(HttpEntity.class),
eq(String.class)
)
);
// triggers the buffer size insertion condition
for (int i = 0; i < 10 * 0.8; i++) {
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
}
// wait for the timer to execute the task again
Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() ->
verify(restTemplate, times(2)).postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(HttpEntity.class),
eq(String.class)
)
);
// wait for the timer to execute the task
Awaitility.await()
.pollInterval(1, TimeUnit.SECONDS)
.atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
}
@Test
@@ -142,22 +141,47 @@ class VictoriaMetricsDataStorageTest {
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the timer to execute its first insertion task
Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() ->
verify(restTemplate, times(1)).postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(HttpEntity.class),
eq(String.class)
)
);
Awaitility.await()
.pollInterval(500, TimeUnit.MILLISECONDS)
.atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the flush interval to be triggered
Awaitility.await().atMost(5, TimeUnit.SECONDS).untilAsserted(() ->
verify(restTemplate, times(2)).postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(),
eq(String.class)
));
// wait for the flush interval to be triggered again
Awaitility.await()
.pollInterval(500, TimeUnit.MILLISECONDS)
.atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(2));
}
@Test
void testMultiThreadSaveDataBySize() {
int threadCount = 100;
int bufferSize = 10;
int writeSize = (int) (bufferSize * 0.8);
// verify insert process for buffer size, with the flush interval defined as an unreachable state
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
bufferSize, Integer.MAX_VALUE, new VictoriaMetricsProperties.Compression(false)));
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
for (int i = 0; i < threadCount; i++) {
new Thread(() -> {
// triggers the buffer size insertion condition
for (int j = 0; j < writeSize; j++) {
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
}
}).start();
}
// wait for the timer to execute the task
Awaitility.await()
.pollInterval(3, TimeUnit.SECONDS)
.atMost(15, TimeUnit.SECONDS)
.untilAsserted(() ->
assertThat(postForEntityCount.get())
// minimum flushes: ensure all data is processed (threadCount * writeSize / bufferSize)
.isGreaterThanOrEqualTo(threadCount * writeSize / bufferSize));
}
@AfterEach
+2 -3
View File
@@ -61,9 +61,8 @@ writing documentation, maintaining issues boards, code review, or answering
community questions) to HertzBeat either by contributing to the codebase
of the main website or HertzBeat's GitHub repositories.
- +3 months with light activity and engagement.
- +2 months of medium activity and engagement.
- +1 month with solid activity and engagement.
- 3+ months with activity and engagement.
- 20+ pr coding, document, test or other contributions.
### Quality of contributions
+1 -3
View File
@@ -61,9 +61,7 @@ writing documentation, maintaining issues boards, code review, or answering
community questions) to HertzBeat either by contributing to the codebase
of the main website or HertzBeat's GitHub repositories.
- +5 months with light activity and engagement.
- +4 months of medium activity and engagement.
- +3 month with solid activity and engagement.
- 12+ months with activity and engagement.
### Quality of contributions
+105
View File
@@ -0,0 +1,105 @@
---
id: dolphinscheduler
title: Monitoring Apache DolphinScheduler
sidebar_label: Apache DolphinScheduler
keywords: [ Open Source Monitoring System, Monitor Apache DolphinScheduler ]
---
> Collect monitoring metrics for Apache DolphinScheduler.
## Pre-monitoring operations
> Support Apache DolphinScheduler version 3.3.0 or later
You need to create a token in Apache DolphinScheduler.
Please refer to [Open API](https://dolphinscheduler.apache.org/zh-cn/docs/3.2.2/guide/api/open-api) to create a new token.The main steps are as follows
1. Log in to the Apache DolphinScheduler system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.
2. Select the "Expiration time" (Token validity time), select "User" (choose the specified user to perform the API operation), click "Generate token", copy the Token string, and click "Submit".
## Configuration Parameters
| Parameter Name | Parameter Help Description |
|---------------------|-----------------------------------------------------------------------------------------------------------------|
| Target Host | The monitored endpoint's IPV4, IPV6, or domain name. Note ⚠️ no protocol header (e.g., https://, http://). |
| Task Name | The name that identifies this monitoring task, which needs to be unique. |
| Port | The monitoring port opened by DolphinScheduler, default value: 12345. |
| SSL | Whether SSL is enabled for connecting to DolphinScheduler. |
| Token | Apache DolphinScheduler token string. |
| Query Timeout | Set the timeout for unresponsive queries, in milliseconds (ms), default 6000 ms. |
| Collection Interval | The interval time for periodic data collection, in seconds; the minimum interval that can be set is 30 seconds. |
| Binding Tags | Used for categorizing and managing monitoring resources. |
| Description Notes | Additional identification and description notes for this monitoring; users can add notes here. |
## Collected Metrics
### Metric Set: Master
| Metric Name | Metric Unit | Metric Help Description |
|----------------|---------------|-------------------------|
| host | None | Host |
| port | None | Port |
| serverStatus | None | Server Status |
| processId | None | Process Id |
| runningTime | Day | Running Time |
| cpuUsage | Percentage(%) | CPU Usage |
| memoryUsage | Percentage(%) | Memory Usage |
| diskUsage | Percentage(%) | Disk Usage |
| jvmCpuUsage | Percentage(%) | JVM CPU Usage |
| jvmMemoryUsage | Percentage(%) | JVM Memory Usage |
| jvmHeapUsed | None | JVM Heap Used |
| jvmNonHeapUsed | None | JVM NonHeap Used |
| jvmHeapMax | None | JVM Heap Max |
| jvmNonHeapMax | None | JVM NonHeap Max |
### Metric Set: Worker
| Metric Name | Metric Unit | Metric Help Description |
|------------------|---------------|-------------------------|
| host | None | Host |
| port | None | Port |
| serverStatus | None | Server Status |
| processId | None | Process Id |
| runningTime | Day | Running Time |
| cpuUsage | Percentage(%) | CPU Usage |
| memoryUsage | Percentage(%) | Memory Usage |
| diskUsage | Percentage(%) | Disk Usage |
| jvmCpuUsage | Percentage(%) | JVM CPU Usage |
| jvmMemoryUsage | Percentage(%) | JVM Memory Usage |
| jvmHeapUsed | None | JVM Heap Used |
| jvmNonHeapUsed | None | JVM NonHeap Used |
| jvmHeapMax | None | JVM Heap Max |
| jvmNonHeapMax | None | JVM NonHeap Max |
| workerHostWeight | None | Weight |
| threadPoolUsage | None | Thread Pool Usage |
| workerGroup | None | Worker Group |
### Metric Set: Alert Server
| Metric Name | Metric Unit | Metric Help Description |
|----------------|---------------|-------------------------|
| host | None | Host |
| port | None | Port |
| serverStatus | None | Server Status |
| processId | None | Process Id |
| runningTime | Day | Running Time |
| cpuUsage | Percentage(%) | CPU Usage |
| memoryUsage | Percentage(%) | Memory Usage |
| diskUsage | Percentage(%) | Disk Usage |
| jvmCpuUsage | Percentage(%) | JVM CPU Usage |
| jvmMemoryUsage | Percentage(%) | JVM Memory Usage |
| jvmHeapUsed | None | JVM Heap Used |
| jvmNonHeapUsed | None | JVM NonHeap Used |
| jvmHeapMax | None | JVM Heap Max |
| jvmNonHeapMax | None | JVM NonHeap Max |
### Metric Set: Database
| Metric Name | Metric Unit | Metric Help Description |
|---------------------------|-------------|-----------------------------|
| dbType | None | Database Type |
| state | None | State |
| maxConnections | None | Max Connections |
| threadsConnections | None | Threads Connections |
| threadsRunningConnections | Day | Threads Running Connections |
+71
View File
@@ -0,0 +1,71 @@
---
id: mcp_sse_server
title: MCP SSE Server
sidebar_label: MCP SSE Server
keywords: [MCP, SSE, streaming, server]
---
This page explains how connect to the HertzBeat MCP SSE server. The MCP server auto starts on the default port 1157 when you start the HertzBeat server.
### Overview
- Provides a ServerSent Events (SSE) stream for tool calling.
- Intended for MCP integrations and clients that consume streaming events.
### Connect to the MCP server
Make sure that hertzbeat server is up and running. If you are using any other port than 1157, replace the following accordingly
- URL: `http://localhost:1157/api/sse`
### Authentication
You must authenticate each request using one of the following methods:
- JWT bearer token
- Header: `Authorization: Bearer <your-jwt-token>`
- Basic authentication
- Header: `Authorization: Basic <base64(username:password)>`
### Cursor MCP configuration
Create or edit `.cursor/mcp.json` in your home directory or project root.
Basic auth:
```json
{
"Hertzbeat-MCP": {
"url": "http://localhost:1157/api/sse",
"headers": {
"Authorization": "Basic <base64(username:password)>"
}
}
}
```
JWT bearer:
```json
{
"Hertzbeat-MCP": {
"url": "http://localhost:1157/api/sse",
"headers": {
"Authorization": "Bearer <your-jwt-token>"
}
}
}
```
After saving, reload MCP in Cursor or restart the editor.
### Tools available
- list_monitors: Returns the list of names of all configured monitors.
More tools are coming soon to expand management and query capabilities.
### Notes
- If the connection drops, reconnect using the same headers.
+108 -5
View File
@@ -15,13 +15,13 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo
### Configuration parameter
| Parameter name | Parameter help description |
| Parameter name | Parameter help description |
|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Monitoring Host | Monitored IPV4, IPV6 or domain name. Note⚠️Without protocol header (eg: https://, http://) |
| Monitoring name | Identify the name of this monitoring. The name needs to be unique |
| Port | Port provided by the database. The default is 1521 |
| Query timeout | Set the timeout time when SQL query does not respond to data, unit: ms, default: 3000ms |
| Database name | Database instance name, optional |
| Database name | Database instance name, optionalIf you need to use a dba user, you can fill in like "sys as sysdba". |
| Username | Database connection user name, optional |
| Password | Database connection password, optional |
| URL | Database connection URLoptionalIf configured, the database name, user name, password and other parameters in the URL will overwrite the above configured parameters |
@@ -36,7 +36,6 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo
| Metric name | Metric unit | Metric help description |
|------------------|-------------|-------------------------|
| database_version | none | Database version |
| database_type | none | Database type |
| hostname | none | Host name |
| instance_name | none | Database instance name |
| startup_time | none | Database start time |
@@ -53,13 +52,31 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo
| bytes | MB | Size |
| blocks | none | Number of blocks |
#### Metric setuser_connect
#### Metric settotal_sessions
| Metric name | Metric unit | Metric help description |
|-------------|-------------|---------------------------|
| username | none | Username |
| counts | number | Current connection counts |
#### Metric setactive_sessions
| Metric name | Metric unit | Metric help description |
|-------------|-------------|-------------------------|
| counts | number | Active sessions counts |
#### Metric setbackground_sessions
| Metric name | Metric unit | Metric help description |
|-------------|-------------|----------------------------|
| counts | number | Background sessions counts |
#### Metric setconnection
| Metric name | Metric unit | Metric help description |
|-------------|-------------|-------------------------|
| username | none | User name |
| counts | number | User sessions counts |
#### Metric setperformance
| Metric name | Metric unit | Metric help description |
@@ -67,3 +84,89 @@ keywords: [open source monitoring tool, open source database monitoring tool, mo
| qps | QPS | I/O Requests per second |
| tps | TPS | User transaction per second |
| mbps | MBPS | I/O Megabytes per second |
#### Metric setpercentage
| Metric name | Metric unit | Metric help description |
|-----------------|---------------|-------------------------|
| tablespace_name | none | Tablespace name |
| total | none | Total |
| used | none | Used |
| free | none | Free |
| used_percentage | percentage(%) | Used Percentage |
| free_percentage | percentage(%) | Free Percentage |
#### Metric setprocess
| Metric name | Metric unit | Metric help description |
|---------------|-------------|-------------------------|
| process_count | none | Process count |
#### Metric settransaction
| Metric name | Metric unit | Metric help description |
|-------------|-------------|-------------------------|
| commits | t/s | User Commits Per Sec |
| rollbacks | t/s | User Rollbacks Per Sec |
#### Metric setwait
| Metric name | Metric unit | Metric help description |
|----------------------|-------------|-------------------------|
| concurrent_wait_time | ms | Concurrent Wait Time |
| commit_wait_time | ms | Commit Wait Time |
| app_wait_time | ms | Application Wait Time |
| network_wait_time | ms | Network Wait Time |
| system_io_wait_time | ms | System I/O Wait Time |
| user_io_wait_time | ms | User I/O Wait Time |
| configure_wait_time | ms | Configure Wait Time |
| scheduler_wait_time | ms | Scheduler Wait Time |
#### Metric setcpu_stats
| Metric name | Metric unit | Metric help description |
|-------------|-------------|-------------------------|
| type | none | Type |
| num | none | Num |
#### Metric setmem_stats
| Metric name | Metric unit | Metric help description |
|-------------|-------------|-------------------------|
| type | none | Type |
| num | none | Num |
#### Metric setcache_hit_ratio
| Metric name | Metric unit | Metric help description |
|------------------------|-------------|-------------------------|
| lib_cache_hit_ratio | none | Library Cache Hit Ratio |
| buffer_cache_hit_ratio | none | Buffer Cache Hit Ratio |
#### Metric setslow_query
| Metric name | Metric unit | Metric help description |
|----------------|-------------|-------------------------|
| sql_id | none | SQL ID |
| child_number | none | Child Number |
| executions | none | EXECUTIONS |
| per_secs | seconds | Per Secs |
| cpu_secs | seconds | CPU Secs |
| buffer_gets | none | Buffer Gets |
| disk_reads | none | Disk Reads |
| fetches | none | Fetches |
| parse_calls | none | Parse Calls |
| optimizer_cost | none | Optimizer Cost |
| sql_text | none | SQL Text |
#### Metric setusers
| Metric name | Metric unit | Metric help description |
|---------------------|-------------|---------------------------------------------------------------------------------------------------|
| username | none | User Name |
| account_status | none | Account Status |
| lock_date | none | If the account status is LOCKED, the date and time when the account was locked will be displayed. |
| expiry_date | none | Password Expiry Date |
| expiry_seconds | seconds | Password Validity Period Remaining |
| created | none | Creation Date |
| authentication_type | none | Authentication Type |
@@ -40,9 +40,8 @@ Apache HertzBeat 社区努力追求基于功绩的原则。因此,一旦有人
Committer 的候选人应该持续参与并为 HertzBeat 做出大量的贡献(例如修复漏洞、添加新功能、编写文档、维护问题板、代码审查或回答社区问题),无论是向主网站的代码库还是 HertzBeat 的 GitHub 仓库贡献。
- +3 个月的轻度活动和参与。
- +2 个月的中度活动和参与
- +1 个月的高度活动和参与。
- 3+ months 的活动和参与。
- 20+ pr 的代码,文档,测试等贡献
### 贡献的质量
@@ -40,9 +40,7 @@ Apache HertzBeat 社区努力追求基于功绩的原则。因此,一旦有人
PMC 成员的候选人应该持续参与并为 HertzBeat 做出大量的贡献(例如修复漏洞、添加新功能、编写文档、维护问题板、代码审查或回答社区问题),无论是向主网站的代码库还是 HertzBeat 的 GitHub 仓库贡献。
- +5 个月的轻度活动和参与。
- +4 个月的中度活动和参与。
- +3 个月的高度活动和参与。
- 12+ months 的活动和参与。
### 贡献的质量
@@ -0,0 +1,105 @@
---
id: dolphinscheduler
title: 监控:Apache DolphinScheduler
sidebar_label: Apache DolphinScheduler
keywords: [ 开源监控系统, 监控 Apache DolphinScheduler ]
---
> 对 Apache DolphinScheduler 指标进行采集监控。
## 监控前操作
> 支持 Apache DolphinScheduler v3.3.0 或更高版本
您需在 Apache DolphinScheduler 中创建令牌。
可参考 [API 调用](https://dolphinscheduler.apache.org/zh-cn/docs/3.2.2/guide/api/open-api) 创建一个新令牌,具体步骤如下:
1. 登录 Apache DolphinScheduler 系统,点击 "安全中心",再点击左侧的 "令牌管理",点击 "令牌管理" 创建令牌。
2. 选择 "失效时间" (Token 有效期),选择 "用户" (以指定的用户执行接口操作),点击 "生成令牌" ,拷贝令牌字符串,然后点击 "提交" 。
## 配置参数
| 参数名称 | 参数帮助描述 |
|-----------|------------------------------------------------------|
| 目标Host | 被监控的对端IPV4,IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 |
| 任务名称 | 标识此监控的名称,名称需要保证唯一性。 |
| 端口 | DolphinScheduler开放的监控端口,默认值:12345。 |
| 启用HTTPS | 是否启用HTTPS。 |
| 令牌 | DolphinScheduler 的令牌字符串。 |
| 查询超时时间 | 设置查询未响应数据时的超时时间,单位ms毫秒,默认6000毫秒。 |
| 采集间隔 | 监控周期性采集数据间隔时间,单位秒,可设置的最小间隔为30秒 |
| 绑定标签 | 用于对监控资源进行分类管理。 |
| 描述备注 | 更多标识和描述此监控的备注信息,用户可以在这里备注信息。 |
## 采集指标
### 指标集合:Master
| 指标名称 | 指标单位 | 指标帮助描述 |
|----------------|--------|-----------------|
| host | 无 | 主机地址 |
| port | 无 | 端口 |
| serverStatus | 无 | 状态 |
| processId | 无 | 进程 ID |
| runningTime | 天 | 运行时间 |
| cpuUsage | 百分比(%) | 处理器使用量 |
| memoryUsage | 百分比(%) | 内存使用量 |
| diskUsage | 百分比(%) | 磁盘可用容量 |
| jvmCpuUsage | 百分比(%) | JVM CPU 使用量 |
| jvmMemoryUsage | 百分比(%) | JVM 内存 使用量 |
| jvmHeapUsed | 无 | JVM 已使用的堆内存大小 |
| jvmNonHeapUsed | 无 | JVM 已使用的非堆内存大小 |
| jvmHeapMax | 无 | JVM 配置的最大堆内存大小 |
| jvmNonHeapMax | 无 | JVM 配置的最大非堆内存大小 |
### 指标集合:Worker
| 指标名称 | 指标单位 | 指标帮助描述 |
|------------------|--------|-----------------|
| host | 无 | 主机地址 |
| port | 无 | 端口 |
| serverStatus | 无 | 状态 |
| processId | 无 | 进程 ID |
| runningTime | 天 | 运行时间 |
| cpuUsage | 百分比(%) | CPU使用率 |
| memoryUsage | 百分比(%) | 内存使用率 |
| diskUsage | 百分比(%) | 磁盘可用容量 |
| jvmCpuUsage | 百分比(%) | JVM CPU 使用量 |
| jvmMemoryUsage | 百分比(%) | JVM 内存 使用量 |
| jvmHeapUsed | 无 | JVM 已使用的堆内存大小 |
| jvmNonHeapUsed | 无 | JVM 已使用的非堆内存大小 |
| jvmHeapMax | 无 | JVM 配置的最大堆内存大小 |
| jvmNonHeapMax | 无 | JVM 配置的最大非堆内存大小 |
| workerHostWeight | 无 | 权重 |
| threadPoolUsage | 无 | 线程池使用量 |
| workerGroup | 无 | Worker 组 |
### 指标集合:Alert Server
| 指标名称 | 指标单位 | 指标帮助描述 |
|----------------|--------|-----------------|
| host | 无 | 主机地址 |
| port | 无 | 端口 |
| serverStatus | 无 | 状态 |
| processId | 无 | 进程 ID |
| runningTime | 天 | 运行时间 |
| cpuUsage | 百分比(%) | 处理器使用量 |
| memoryUsage | 百分比(%) | 内存使用量 |
| diskUsage | 百分比(%) | 磁盘可用容量 |
| jvmCpuUsage | 百分比(%) | JVM CPU 使用量 |
| jvmMemoryUsage | 百分比(%) | JVM 内存 使用量 |
| jvmHeapUsed | 无 | JVM 已使用的堆内存大小 |
| jvmNonHeapUsed | 无 | JVM 已使用的非堆内存大小 |
| jvmHeapMax | 无 | JVM 配置的最大堆内存大小 |
| jvmNonHeapMax | 无 | JVM 配置的最大非堆内存大小 |
### 指标:数据库
| 指标名称 | 指标单位 | 指标帮助描述 |
|---------------------------|--------|-----------|
| dbType | 无 | 数据库类型 |
| state | 无 | 状态 |
| maxConnections | 无 | 最大连接数 |
| threadsConnections | 无 | 当前连接数 |
| threadsRunningConnections | 天 | 当前活跃连接数 |
@@ -15,14 +15,14 @@ keywords: [开源监控系统, 开源数据库监控, Oracle数据库监控]
### 配置参数
| 参数名称 | 参数帮助描述 |
| 参数名称 | 参数帮助描述 |
|--------|------------------------------------------------------|
| 监控Host | 被监控的对端IPV4,IPV6或域名。注意⚠️不带协议头(eg: https://, http://)。 |
| 任务名称 | 标识此监控的名称,名称需要保证唯一性。 |
| 端口 | 数据库对外提供的端口,默认为1521。 |
| 查询超时时间 | 设置SQL查询未响应数据时的超时时间,单位ms毫秒,默认3000毫秒。 |
| 数据库名称 | 数据库实例名称,可选。 |
| 用户名 | 数据库连接用户名,可选 |
| 用户名 | 数据库连接用户名,可选。如果使用 sys 用户,可填写成"sys as sysdba" |
| 密码 | 数据库连接密码,可选 |
| URL | 数据库连接URL,可选,若配置,则URL里面的数据库名称,用户名密码等参数会覆盖上面配置的参数 |
| 采集间隔 | 监控周期性采集数据间隔时间,单位秒,可设置的最小间隔为30秒 |
@@ -31,18 +31,17 @@ keywords: [开源监控系统, 开源数据库监控, Oracle数据库监控]
### 采集指标
#### 指标集合:basic
#### 指标集合:基本信息
| 指标名称 | 指标单位 | 指标帮助描述 |
|------------------|------|---------|
| database_version | 无 | 数据库版本 |
| database_type | 无 | 数据库类型 |
| hostname | 无 | 主机名称 |
| instance_name | 无 | 数据库实例名称 |
| startup_time | 无 | 数据库启动时间 |
| status | 无 | 数据库状态 |
#### 指标集合:tablespace
#### 指标集合:表空间
| 指标名称 | 指标单位 | 指标帮助描述 |
|-----------------|------|---------|
@@ -51,19 +50,123 @@ keywords: [开源监控系统, 开源数据库监控, Oracle数据库监控]
| tablespace_name | 无 | 所属表空间名称 |
| status | 无 | 状态 |
| bytes | MB | 大小 |
| blocks | | 区块数量 |
| blocks | | 区块数量 |
#### 指标集合:user_connect
#### 指标集合:会话总数
| 指标名称 | 指标单位 | 指标帮助描述 |
| 指标名称 | 指标单位 | 指标帮助描述 |
|-------|------|--------|
| count | 无 | 总数 |
#### 指标集合:活动会话
| 指标名称 | 指标单位 | 指标帮助描述 |
|-------|------|--------|
| count | 无 | 总数 |
#### 指标集合:后台会话
| 指标名称 | 指标单位 | 指标帮助描述 |
|-------|------|--------|
| count | 无 | 总数 |
#### 指标集合:连接
| 指标名称 | 指标单位 | 指标帮助描述 |
|----------|------|--------|
| username | 无 | 用户名 |
| counts | 个数 | 当前连接数量 |
| count | | 总数 |
#### 指标集合:performance
#### 指标集合:性能
| 指标名称 | 指标单位 | 指标帮助描述 |
|------|------|---------------------------------------|
| qps | QPS | I/O Requests per Second 每秒IO请求数量 |
| tps | TPS | User Transaction Per Sec 每秒用户事物处理数量 |
| mbps | MBPS | I/O Megabytes per Second 每秒 I/O 兆字节数量 |
#### 指标集合:表空间百分比
| 指标名称 | 指标单位 | 指标帮助描述 |
|-----------------|--------|--------|
| tablespace_name | 无 | 表空间名 |
| total | 无 | 全部 |
| used | 无 | 已用 |
| free | 无 | 空闲 |
| used_percentage | 百分比(%) | 已用百分比 |
| free_percentage | 百分比(%) | 空闲百分比 |
#### 指标集合:进程
| 指标名称 | 指标单位 | 指标帮助描述 |
|---------------|------|--------|
| process_count | 无 | 进程数 |
#### 指标集合:事务
| 指标名称 | 指标单位 | 指标帮助描述 |
|-----------|------|--------|
| commits | t/s | 提交数 |
| rollbacks | t/s | 回滚数 |
#### 指标集合:等待
| 指标名称 | 指标单位 | 指标帮助描述 |
|----------------------|--------|-----------|
| concurrent_wait_time | 毫秒(ms) | 并发等待时间 |
| commit_wait_time | 毫秒(ms) | 提交等待时间 |
| app_wait_time | 毫秒(ms) | 应用等待时间 |
| network_wait_time | 毫秒(ms) | 网络等待时间 |
| system_io_wait_time | 毫秒(ms) | 系统I/O等待时间 |
| user_io_wait_time | 毫秒(ms) | 用户I/O等待时间 |
| configure_wait_time | 毫秒(ms) | 配置等待时间 |
| scheduler_wait_time | 毫秒(ms) | 调度等待时间 |
#### 指标集合:CPU 状态
| 指标名称 | 指标单位 | 指标帮助描述 |
|------|------|--------|
| type | 无 | 类型 |
| num | 无 | 数量 |
#### 指标集合:内存状态
| 指标名称 | 指标单位 | 指标帮助描述 |
|------|------|--------|
| type | 无 | 类型 |
| num | 无 | 数量 |
#### 指标集合:缓存命中率
| 指标名称 | 指标单位 | 指标帮助描述 |
|------------------------|------|----------|
| lib_cache_hit_ratio | 无 | 库缓存命中率 |
| buffer_cache_hit_ratio | 无 | 缓冲区缓存命中率 |
#### 指标集合:慢查询
| 指标名称 | 指标单位 | 指标帮助描述 |
|----------------|------|--------|
| sql_id | 无 | sql 主键 |
| child_number | 无 | 子编号 |
| executions | 次 | 执行数 |
| per_secs | 秒 | 每秒执行数 |
| cpu_secs | 秒 | 每秒 CPU |
| buffer_gets | 无 | 获得的缓冲区 |
| disk_reads | 无 | 磁盘读取 |
| fetches | 无 | 获取数量 |
| parse_calls | 无 | 解析调用 |
| optimizer_cost | 无 | 优化器成本 |
| sql_text | 无 | SQL 文本 |
#### 指标集合:用户信息
| 指标名称 | 指标单位 | 指标帮助描述 |
|---------------------|------|------------------------------|
| username | 无 | 用户名 |
| account_status | 无 | 账号状态 |
| lock_date | 无 | 如果账户状态为 LOCKED,则显示锁定账户的日期和时间 |
| expiry_date | 无 | 密码的失效时间 |
| expiry_seconds | 秒 | 密码剩余有效时间,小于 0 表示已失效 |
| created | 无 | 创建时间 |
| authentication_type | 无 | 认证类型 |
+3 -1
View File
@@ -241,7 +241,8 @@
"help/presto",
"help/seatunnel",
"help/spark",
"help/yarn"
"help/yarn",
"help/dolphinscheduler"
]
},
{
@@ -273,6 +274,7 @@
"help/plugin",
"help/time_expression",
"help/grafana_dashboard",
"help/mcp_sse_server",
"help/collector",
"help/ai_config",
"help/issue"
+2
View File
@@ -92,6 +92,8 @@
<module>hertzbeat-e2e</module>
<module>hertzbeat-base</module>
<module>hertzbeat-mcp</module>
<module>hertzbeat-ai-agent</module>
</modules>
<properties>