Merge managed setup runtime into 2.0.0

This commit is contained in:
Logic
2026-08-13 23:20:54 +08:00
698 changed files with 76978 additions and 755 deletions
@@ -19,7 +19,6 @@
package org.apache.hertzbeat.ai.config;
import com.openai.client.OpenAIClient;
import jakarta.annotation.PostConstruct;
import java.time.Duration;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@@ -58,8 +57,7 @@ public class LlmConfig {
this.applicationContext = applicationContext;
}
@PostConstruct
public void registerInitialChatClient() {
void registerInitialChatClient() {
registerChatClient();
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.ai.config;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/** Registers the configured chat client only after the business runtime opens. */
@Component
@ConditionalOnNormalBusinessRuntime
public final class LlmConfigInitializer implements CommandLineRunner {
private final LlmConfig llmConfig;
public LlmConfigInitializer(LlmConfig llmConfig) {
this.llmConfig = llmConfig;
}
@Override
public void run(String... args) {
llmConfig.registerInitialChatClient();
}
}
@@ -30,6 +30,7 @@ import org.apache.hertzbeat.ai.sop.registry.SkillRegistry;
import org.apache.hertzbeat.ai.utils.SopMessageUtil;
import org.apache.hertzbeat.common.entity.ai.ChatMessage;
import org.apache.hertzbeat.common.entity.ai.SopSchedule;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
@@ -43,6 +44,7 @@ import org.springframework.stereotype.Component;
*/
@Slf4j
@Component
@ConditionalOnNormalBusinessRuntime
@EnableScheduling
public class SopScheduleExecutor {
@@ -20,8 +20,8 @@ package org.apache.hertzbeat.alert.calculate.periodic;
import static org.apache.hertzbeat.common.constants.CommonConstants.LOG_ALERT_THRESHOLD_TYPE_PERIODIC;
import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC;
import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -37,28 +37,30 @@ import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* Periodic Alert Rule Scheduler
*/
@Slf4j
@Component
public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean {
public class PeriodicAlertRuleScheduler {
private final MetricsPeriodicAlertCalculator metricsCalculator;
private final LogPeriodicAlertCalculator logCalculator;
private final TracePeriodicAlertCalculator traceCalculator;
private final AlertDefineDao alertDefineDao;
private final ScheduledExecutorService scheduledExecutor;
private final ExecutorService periodicExecutor;
private final Semaphore periodicPermits;
private final boolean virtualThreadsEnabled;
private ScheduledExecutorService scheduledExecutor;
private ExecutorService periodicExecutor;
private Semaphore periodicPermits;
private final VirtualThreadProperties virtualThreadProperties;
private boolean virtualThreadsEnabled;
private final Map<Long, ScheduledTaskState> scheduledTasks;
private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate();
private boolean maintenancePaused;
@Autowired
public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator,
@@ -70,10 +72,15 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
this.logCalculator = logCalculator;
this.traceCalculator = traceCalculator;
this.alertDefineDao = alertDefineDao;
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("Scheduled periodic alert threshold has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
this.virtualThreadProperties = virtualThreadProperties == null
? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.scheduledTasks = new ConcurrentHashMap<>();
}
synchronized void start() {
if (scheduledExecutor != null) {
return;
}
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler((thread, throwable) -> {
log.error("Scheduled periodic alert threshold has uncaughtException.");
@@ -82,49 +89,18 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
.setDaemon(true)
.setNameFormat("periodic-alert-threshold-worker-%d")
.build();
this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory);
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.virtualThreadsEnabled = properties.enabled();
int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs());
scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory);
virtualThreadsEnabled = virtualThreadProperties.enabled();
int maxConcurrentPeriodicTasks = Math.max(
1, virtualThreadProperties.alerter().periodicMaxConcurrentJobs());
this.periodicExecutor = virtualThreadsEnabled
? Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("periodic-alert-task-", 0)
.uncaughtExceptionHandler(handler)
.uncaughtExceptionHandler((thread, throwable) ->
log.error("Periodic alert task failed", throwable))
.factory())
: null;
this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null;
this.scheduledTasks = new ConcurrentHashMap<>();
}
public void cancelSchedule(Long ruleId) {
if (ruleId == null) {
return;
}
ScheduledTaskState state = scheduledTasks.remove(ruleId);
if (state != null) {
state.cancel();
}
}
public void updateSchedule(AlertDefine rule) {
if (rule == null || rule.getId() == null) {
log.error("Alert rule is null or rule id is null.");
return;
}
cancelSchedule(rule.getId());
if (isPeriodicRule(rule.getType())) {
ScheduledTaskState state = new ScheduledTaskState(rule);
ScheduledFuture<?> future = scheduledExecutor.scheduleAtFixedRate(
virtualThreadsEnabled ? state::trigger : () -> executeRule(rule),
0, rule.getPeriod(), TimeUnit.SECONDS);
state.setScheduledFuture(future);
scheduledTasks.put(rule.getId(), state);
}
}
@Override
public void run(String... args) throws Exception {
log.info("Starting periodic alert rule scheduler...");
List<AlertDefine> metricsPeriodicRules = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC);
List<AlertDefine> logPeriodicRules = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_PERIODIC);
@@ -139,13 +115,50 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
}
}
@Override
public void destroy() {
synchronized void stop() {
scheduledTasks.values().forEach(ScheduledTaskState::cancel);
scheduledTasks.clear();
scheduledExecutor.shutdownNow();
if (scheduledExecutor != null) {
scheduledExecutor.shutdownNow();
scheduledExecutor = null;
}
if (periodicExecutor != null) {
periodicExecutor.shutdownNow();
periodicExecutor = null;
}
periodicPermits = null;
}
public synchronized void cancelSchedule(Long ruleId) {
if (ruleId == null || scheduledExecutor == null) {
return;
}
ScheduledTaskState state = scheduledTasks.remove(ruleId);
if (state != null) {
state.cancel();
}
}
public synchronized void updateSchedule(AlertDefine rule) {
if (rule == null || rule.getId() == null) {
log.error("Alert rule is null or rule id is null.");
return;
}
if (scheduledExecutor == null) {
return;
}
cancelSchedule(rule.getId());
if (isPeriodicRule(rule.getType())) {
ScheduledExecutorService currentScheduledExecutor = scheduledExecutor;
ExecutorService currentPeriodicExecutor = periodicExecutor;
Semaphore currentPeriodicPermits = periodicPermits;
ScheduledTaskState state = new ScheduledTaskState(
rule, currentPeriodicExecutor, currentPeriodicPermits);
ScheduledFuture<?> future = currentScheduledExecutor.scheduleAtFixedRate(
state::trigger,
0, rule.getPeriod(), TimeUnit.SECONDS);
state.setScheduledFuture(future);
scheduledTasks.put(rule.getId(), state);
}
}
@@ -159,6 +172,48 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
}
}
public synchronized void pauseAdmission() {
maintenancePaused = true;
maintenanceGate.pauseAdmission();
}
public void awaitDrained(long timeoutNanos) throws InterruptedException, java.util.concurrent.TimeoutException {
maintenanceGate.awaitDrained(timeoutNanos);
}
public void resumeAdmission() {
List<ScheduledTaskState> states;
synchronized (this) {
if (!maintenancePaused) {
return;
}
maintenanceGate.resumeAdmission();
maintenancePaused = false;
states = new ArrayList<>(scheduledTasks.values());
}
states.forEach(ScheduledTaskState::resumeMissed);
}
private synchronized WorkAdmissionGate.Permit acquireTriggerPermit(ScheduledTaskState state) {
if (maintenancePaused) {
state.markMissed();
return null;
}
return maintenanceGate.tryAcquire();
}
void beforeRuleTrigger(AlertDefine rule) {
}
private void executeRuleWithPermit(AlertDefine rule, WorkAdmissionGate.Permit permit) {
if (permit == null) {
return;
}
try (permit) {
executeRule(rule);
}
}
private boolean isPeriodicRule(String type) {
return METRIC_ALERT_THRESHOLD_TYPE_PERIODIC.equals(type)
|| LOG_ALERT_THRESHOLD_TYPE_PERIODIC.equals(type)
@@ -168,35 +223,59 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
private final class ScheduledTaskState {
private final AlertDefine rule;
private final ExecutorService taskExecutor;
private final Semaphore taskPermits;
private ScheduledFuture<?> scheduledFuture;
private Future<?> runningFuture;
private boolean running;
private boolean pending;
private WorkAdmissionGate.Permit pendingPermit;
private boolean cancelled;
private boolean missedWhilePaused;
private ScheduledTaskState(AlertDefine rule) {
private ScheduledTaskState(AlertDefine rule, ExecutorService taskExecutor, Semaphore taskPermits) {
this.rule = rule;
this.taskExecutor = taskExecutor;
this.taskPermits = taskPermits;
}
private synchronized void setScheduledFuture(ScheduledFuture<?> scheduledFuture) {
this.scheduledFuture = scheduledFuture;
}
private synchronized void trigger() {
if (cancelled) {
private void trigger() {
beforeRuleTrigger(rule);
WorkAdmissionGate.Permit permit = acquireTriggerPermit(this);
if (permit == null) {
return;
}
if (running) {
pending = true;
return;
synchronized (this) {
if (cancelled) {
permit.close();
return;
}
if (running) {
if (!pending) {
pending = true;
pendingPermit = permit;
} else {
permit.close();
}
return;
}
running = true;
}
running = true;
submitLocked();
submit(permit);
}
private synchronized void cancel() {
cancelled = true;
pending = false;
if (pendingPermit != null) {
pendingPermit.close();
pendingPermit = null;
}
missedWhilePaused = false;
ScheduledFuture<?> periodicFuture = scheduledFuture;
Future<?> currentFuture = runningFuture;
if (periodicFuture != null) {
@@ -207,33 +286,46 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
}
}
private void submitLocked() {
private void submit(WorkAdmissionGate.Permit permit) {
if (taskExecutor == null) {
runTask(permit);
return;
}
try {
runningFuture = periodicExecutor.submit(() -> {
boolean permitAcquired = false;
try {
periodicPermits.acquire();
permitAcquired = true;
if (!Thread.currentThread().isInterrupted()) {
executeRule(rule);
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e);
} finally {
if (permitAcquired) {
periodicPermits.release();
}
onComplete();
}
});
runningFuture = taskExecutor.submit(() -> runTask(permit));
} catch (RuntimeException e) {
running = false;
permit.close();
throw e;
}
}
private void runTask(WorkAdmissionGate.Permit permit) {
boolean concurrencyPermitAcquired = false;
try {
if (taskPermits != null) {
taskPermits.acquire();
concurrencyPermitAcquired = true;
}
if (!Thread.currentThread().isInterrupted()) {
executeRuleWithPermit(rule, permit);
permit = null;
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e);
} finally {
if (permit != null) {
permit.close();
}
if (concurrencyPermitAcquired) {
taskPermits.release();
}
onComplete();
}
}
private synchronized void onComplete() {
runningFuture = null;
if (cancelled) {
@@ -246,7 +338,25 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
return;
}
pending = false;
submitLocked();
WorkAdmissionGate.Permit nextPermit = pendingPermit;
pendingPermit = null;
submit(nextPermit);
}
private synchronized void markMissed() {
if (!cancelled) {
missedWhilePaused = true;
}
}
private void resumeMissed() {
synchronized (this) {
if (!missedWhilePaused || cancelled) {
return;
}
missedWhilePaused = false;
}
trigger();
}
}
}
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.calculate.periodic;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/** Owns scheduler threads and database startup reads only in normal business runtime. */
@Component
@ConditionalOnNormalBusinessRuntime
public final class PeriodicAlertRuleSchedulerLifecycle implements CommandLineRunner, DisposableBean {
private final PeriodicAlertRuleScheduler scheduler;
public PeriodicAlertRuleSchedulerLifecycle(PeriodicAlertRuleScheduler scheduler) {
this.scheduler = scheduler;
}
@Override
public void run(String... args) {
scheduler.start();
}
@Override
public void destroy() {
scheduler.stop();
}
}
@@ -137,8 +137,10 @@ public class MetricsRealTimeAlertCalculator {
continue;
}
backoff.reset();
calculate(metricsData);
// The telemetry handoff precedes alert reduction so maintenance backpressure cannot
// occupy every calculator before later samples reach storage.
dataQueue.sendMetricsDataToStorage(metricsData);
calculate(metricsData);
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
} catch (CommonDataQueueUnknownException ue) {
@@ -25,6 +25,7 @@ import org.apache.hertzbeat.alert.calculate.realtime.window.LogWorker;
import org.apache.hertzbeat.alert.calculate.realtime.window.TimeService;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.hertzbeat.common.util.BackoffUtils;
import org.apache.hertzbeat.common.util.ExponentialBackoff;
@@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit;
* 4. Distributing logs to workers
*/
@Component
@ConditionalOnNormalBusinessRuntime
@Slf4j
public class WindowedLogRealTimeAlertCalculator implements Runnable {
@@ -23,6 +23,7 @@ import org.apache.hertzbeat.alert.calculate.JexlExprCalculator;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -38,6 +39,7 @@ import java.util.Map;
*/
@Slf4j
@Component
@ConditionalOnNormalBusinessRuntime
public class LogWorker {
private static final String LOG_PREFIX = "log";
@@ -120,4 +122,4 @@ public class LogWorker {
}
return System.currentTimeMillis();
}
}
}
@@ -24,6 +24,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.stereotype.Component;
import jakarta.annotation.PreDestroy;
@@ -42,6 +43,7 @@ import java.util.concurrent.atomic.AtomicLong;
* 3. Broadcasting watermarks to all subscribers (WindowAggregator)
*/
@Component
@ConditionalOnNormalBusinessRuntime
@Slf4j
public class TimeService {
@@ -24,6 +24,7 @@ import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.stereotype.Component;
import jakarta.annotation.PreDestroy;
@@ -49,6 +50,7 @@ import java.util.concurrent.TimeUnit;
* 4. Sending closed windows to AlarmEvaluator
*/
@Component
@ConditionalOnNormalBusinessRuntime
@Slf4j
public class WindowAggregator implements TimeService.WatermarkListener, Runnable {
@@ -106,18 +106,29 @@ public class AlertNoticeDispatch {
return Optional.ofNullable(noticeConfigService.getReceiverFilterRule(alert));
}
public void dispatchAlarm(GroupAlert groupAlert) {
if (groupAlert != null) {
// Determining alarm type storage
GroupAlert storedGroupAlert = alertStoreHandler.store(groupAlert);
// Notice distribution
sendNotify(storedGroupAlert);
// Execute the plugin if enable (Compatible with old version plugins, will be removed in later versions)
pluginRunner.pluginExecute(Plugin.class, plugin -> plugin.alert(storedGroupAlert));
// Execute the plugin if enable with params
pluginRunner.pluginExecute(PostAlertPlugin.class, (afterAlertPlugin, pluginContext) -> afterAlertPlugin.execute(storedGroupAlert, pluginContext));
// Send alert to the sse client
emitterManager.broadcast(JsonUtil.toJson(storedGroupAlert));
public boolean dispatchAlarm(GroupAlert groupAlert) {
if (groupAlert == null) {
return false;
}
GroupAlert storedGroupAlert = alertStoreHandler.store(groupAlert);
dispatchAfterStore(storedGroupAlert);
return true;
}
private void dispatchAfterStore(GroupAlert storedGroupAlert) {
runAfterStore(() -> sendNotify(storedGroupAlert), "notice");
runAfterStore(() -> pluginRunner.pluginExecute(
Plugin.class, plugin -> plugin.alert(storedGroupAlert)), "legacy-plugin");
runAfterStore(() -> pluginRunner.pluginExecute(PostAlertPlugin.class,
(plugin, context) -> plugin.execute(storedGroupAlert, context)), "post-plugin");
runAfterStore(() -> emitterManager.broadcast(JsonUtil.toJson(storedGroupAlert)), "sse");
}
private void runAfterStore(Runnable action, String stage) {
try {
action.run();
} catch (RuntimeException exception) {
log.warn("Post-store alert dispatch failed at stage: {}", stage);
}
}
@@ -54,6 +54,9 @@ public class EmailAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
@Value("${spring.mail.username:demo}")
private String username;
@Value("${hertzbeat.mail.from-address:${spring.mail.username:demo}}")
private String fromAddress;
@Value("${spring.mail.password:demo}")
private String password;
@@ -82,7 +85,7 @@ public class EmailAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
try {
// get sender
JavaMailSenderImpl sender = (JavaMailSenderImpl) javaMailSender;
String fromUsername = username;
String fromUsername = fromAddress;
try {
boolean useDatabase = false;
GeneralConfig emailConfig = generalConfigDao.findByType(TYPE);
@@ -18,12 +18,17 @@
package org.apache.hertzbeat.alert.reduce;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.beans.factory.DisposableBean;
@@ -44,6 +49,14 @@ public class AlarmCommonReduce implements DisposableBean {
private final ApplicationEventPublisher eventPublisher;
private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate();
private final ReentrantLock maintenanceLock = new ReentrantLock(true);
private final Deque<Runnable> deferredTasks = new ArrayDeque<>();
private boolean stopped;
public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) {
this(alarmGroupReduce, VirtualThreadProperties.defaults(), event -> { });
}
@@ -62,6 +75,12 @@ public class AlarmCommonReduce implements DisposableBean {
this.workerExecutor = initWorkExecutor(properties);
}
AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, ManagedExecutor workerExecutor) {
this.alarmGroupReduce = alarmGroupReduce;
this.workerExecutor = workerExecutor;
this.eventPublisher = event -> { };
}
private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) {
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("alerter-reduce-worker has uncaughtException.");
@@ -87,11 +106,11 @@ public class AlarmCommonReduce implements DisposableBean {
public void reduceAndSendAlarm(SingleAlert alert) {
workerExecutor.execute(reduceAlarmTask(alert));
submitOrDefer(reduceAlarmTask(alert));
}
public void reduceAndSendAlarmGroup(Map<String, String> groupLabels, List<SingleAlert> alerts) {
workerExecutor.execute(() -> {
submitOrDefer(() -> {
try {
// Generate alert fingerprint
for (SingleAlert alert : alerts) {
@@ -107,6 +126,87 @@ public class AlarmCommonReduce implements DisposableBean {
});
}
public void pauseAdmission() {
maintenanceLock.lock();
try {
maintenanceGate.pauseAdmission();
} finally {
maintenanceLock.unlock();
}
}
public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException {
maintenanceGate.awaitDrained(timeoutNanos);
}
public void resumeAdmission() {
maintenanceLock.lock();
try {
if (stopped) {
return;
}
while (!deferredTasks.isEmpty()) {
Runnable deferred = deferredTasks.peekFirst();
WorkAdmissionGate.Permit permit = maintenanceGate.reserveReplay();
if (permit == null) {
return;
}
submitAdmitted(deferred, permit);
deferredTasks.removeFirst();
}
maintenanceGate.resumeAdmission();
} finally {
maintenanceLock.unlock();
}
}
private void submitOrDefer(Runnable task) {
maintenanceLock.lock();
try {
if (stopped) {
return;
}
WorkAdmissionGate.Permit permit = maintenanceGate.tryAcquire();
if (permit != null) {
beforeAdmittedSubmission();
submitAdmitted(task, permit);
return;
}
deferredTasks.addLast(task);
} finally {
maintenanceLock.unlock();
}
}
void beforeAdmittedSubmission() {
}
boolean hasQueuedMaintenanceThread(Thread thread) {
return maintenanceLock.hasQueuedThread(thread);
}
int deferredTaskCount() {
maintenanceLock.lock();
try {
return deferredTasks.size();
} finally {
maintenanceLock.unlock();
}
}
private void submitAdmitted(Runnable task, WorkAdmissionGate.Permit permit) {
try {
workerExecutor.execute(() -> {
try (permit) {
task.run();
}
});
} catch (RuntimeException exception) {
permit.close();
throw exception;
}
}
Runnable reduceAlarmTask(SingleAlert alert) {
return () -> {
try {
@@ -138,6 +238,14 @@ public class AlarmCommonReduce implements DisposableBean {
@Override
public void destroy() {
maintenanceLock.lock();
try {
maintenanceGate.stop();
deferredTasks.clear();
stopped = true;
} finally {
maintenanceLock.unlock();
}
workerExecutor.close();
}
}
@@ -31,6 +31,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@@ -93,57 +94,100 @@ public class AlarmGroupReduce implements DisposableBean {
*/
private final Map<String, GroupAlertCache> groupCacheMap;
private final ScheduledExecutorService scheduledExecutor;
private final AlertGroupConvergeDao alertGroupConvergeDao;
private final VirtualThreadProperties virtualThreadProperties;
private ScheduledExecutorService scheduledExecutor;
private final ExecutorService workerExecutor;
private ExecutorService workerExecutor;
private final ScheduledDispatchTask checkTask;
private ScheduledDispatchTask checkTask;
public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) {
this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true);
this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults());
}
@Autowired
public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
VirtualThreadProperties virtualThreadProperties) {
this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true);
}
AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
VirtualThreadProperties virtualThreadProperties, boolean autoStart) {
this.alarmInhibitReduce = alarmInhibitReduce;
this.groupDefines = new ConcurrentHashMap<>(8);
this.groupCacheMap = new ConcurrentHashMap<>(8);
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.scheduledExecutor = createScheduler();
this.workerExecutor = createVirtualExecutor(properties);
this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups);
List<AlertGroupConverge> groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue();
refreshGroupDefines(groupConverges);
if (autoStart) {
startCheckAndSendGroups();
}
this.alertGroupConvergeDao = alertGroupConvergeDao;
this.virtualThreadProperties = virtualThreadProperties == null
? VirtualThreadProperties.defaults() : virtualThreadProperties;
}
private void startCheckAndSendGroups() {
scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL,
synchronized void start() {
if (scheduledExecutor != null) {
return;
}
scheduledExecutor = createScheduler();
workerExecutor = createVirtualExecutor(virtualThreadProperties);
ScheduledDispatchTask currentCheckTask =
new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups);
checkTask = currentCheckTask;
refreshGroupDefines(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue());
startCheckAndSendGroups(currentCheckTask);
}
private void startCheckAndSendGroups(ScheduledDispatchTask currentCheckTask) {
scheduledExecutor.scheduleAtFixedRate(currentCheckTask::dispatch, 10000, CHECK_INTERVAL,
TimeUnit.MILLISECONDS);
}
void dispatchCheckAndSendGroups() {
checkTask.dispatch();
synchronized void dispatchCheckAndSendGroups() {
if (checkTask != null) {
checkTask.dispatch();
}
}
void beforeCheckAndSendGroupsRun() {
}
public void pauseAdmission() {
ScheduledDispatchTask currentTask;
synchronized (this) {
currentTask = checkTask;
}
if (currentTask != null) {
currentTask.pauseAdmission();
}
}
public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException {
ScheduledDispatchTask currentTask;
synchronized (this) {
currentTask = checkTask;
}
if (currentTask != null) {
currentTask.awaitDrained(timeoutNanos);
}
}
public void resumeAdmission() {
ScheduledDispatchTask currentTask;
synchronized (this) {
currentTask = checkTask;
}
if (currentTask != null) {
currentTask.resumeAdmission();
}
}
@Override
public void destroy() {
scheduledExecutor.shutdownNow();
public synchronized void destroy() {
if (checkTask != null) {
checkTask.cancel();
}
if (scheduledExecutor != null) {
scheduledExecutor.shutdownNow();
scheduledExecutor = null;
}
if (workerExecutor != null) {
workerExecutor.shutdownNow();
workerExecutor = null;
}
checkTask = null;
}
private ScheduledExecutorService createScheduler() {
@@ -178,8 +222,6 @@ public class AlarmGroupReduce implements DisposableBean {
groupCacheMap.forEach((groupKey, cache) -> {
if (shouldSendGroup(cache, now)) {
sendGroupAlert(cache);
cache.setLastSendTime(now);
cache.getAlertFingerprints().clear();
}
});
} catch (Exception e) {
@@ -276,43 +318,50 @@ public class AlarmGroupReduce implements DisposableBean {
if (shouldSendGroupImmediately(cache)) {
sendGroupAlert(cache);
cache.setLastSendTime(System.currentTimeMillis());
cache.getAlertFingerprints().clear();
}
}
private void sendGroupAlert(GroupAlertCache cache) {
if (cache.getAlertFingerprints().isEmpty()) {
return;
}
long now = System.currentTimeMillis();
String status = determineGroupStatus(cache.getAlertFingerprints().values());
// For firing alerts, check repeat interval
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
long repeatInterval = ruleConfig.getRepeatInterval() != null
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
// Skip if within repeat interval
if (cache.getLastRepeatTime() > 0
&& now - cache.getLastRepeatTime() < repeatInterval) {
synchronized (cache) {
Map<String, SingleAlert> snapshot = new HashMap<>(cache.getAlertFingerprints());
if (snapshot.isEmpty()) {
return;
}
cache.setLastRepeatTime(now);
long now = System.currentTimeMillis();
String status = determineGroupStatus(snapshot.values());
// For firing alerts, check repeat interval without consuming the retained snapshot.
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
long repeatInterval = ruleConfig.getRepeatInterval() != null
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
if (cache.getLastRepeatTime() > 0
&& now - cache.getLastRepeatTime() < repeatInterval) {
return;
}
}
GroupAlert groupAlert = GroupAlert.builder()
.groupKey(cache.getGroupKey())
.groupLabels(cache.getGroupLabels())
.commonLabels(extractCommonLabels(snapshot.values()))
.commonAnnotations(extractCommonAnnotations(snapshot.values()))
.alerts(new ArrayList<>(snapshot.values()))
.status(status)
.build();
if (!alarmInhibitReduce.inhibitAlarm(groupAlert)) {
return;
}
snapshot.forEach((fingerprint, alert) ->
cache.getAlertFingerprints().remove(fingerprint, alert));
cache.setLastSendTime(now);
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
cache.setLastRepeatTime(now);
}
}
GroupAlert groupAlert = GroupAlert.builder()
.groupKey(cache.getGroupKey())
.groupLabels(cache.getGroupLabels())
.commonLabels(extractCommonLabels(cache.getAlertFingerprints().values()))
.commonAnnotations(extractCommonAnnotations(cache.getAlertFingerprints().values()))
.alerts(new ArrayList<>(cache.getAlertFingerprints().values()))
.status(status)
.build();
alarmInhibitReduce.inhibitAlarm(groupAlert);
}
private boolean shouldSendGroup(GroupAlertCache cache, long now) {
@@ -414,6 +463,12 @@ public class AlarmGroupReduce implements DisposableBean {
private int pendingRuns;
private boolean cancelled;
private boolean paused;
private boolean missedWhilePaused;
private ScheduledDispatchTask(ExecutorService executor, Runnable task) {
this.executor = executor;
this.task = task;
@@ -422,6 +477,13 @@ public class AlarmGroupReduce implements DisposableBean {
private void dispatch() {
boolean shouldSchedule;
synchronized (this) {
if (cancelled) {
return;
}
if (paused) {
missedWhilePaused = true;
return;
}
pendingRuns++;
shouldSchedule = !running;
if (shouldSchedule) {
@@ -452,14 +514,69 @@ public class AlarmGroupReduce implements DisposableBean {
private void scheduleNextIfNeeded() {
boolean shouldSchedule;
synchronized (this) {
if (cancelled) {
pendingRuns = 0;
running = false;
notifyAll();
return;
}
pendingRuns = Math.max(0, pendingRuns - 1);
shouldSchedule = pendingRuns > 0;
if (!shouldSchedule) {
running = false;
notifyAll();
return;
}
}
scheduleRun();
}
private synchronized void cancel() {
cancelled = true;
pendingRuns = 0;
missedWhilePaused = false;
notifyAll();
}
private synchronized void pauseAdmission() {
paused = true;
missedWhilePaused |= pendingRuns > 1;
pendingRuns = running ? 1 : 0;
}
private synchronized void awaitDrained(long timeoutNanos)
throws InterruptedException, TimeoutException {
long remainingNanos = timeoutNanos;
long startedNanos = System.nanoTime();
while (running) {
if (remainingNanos <= 0) {
throw new TimeoutException();
}
TimeUnit.NANOSECONDS.timedWait(this, remainingNanos);
long elapsedNanos = System.nanoTime() - startedNanos;
if (elapsedNanos <= 0) {
remainingNanos = timeoutNanos;
} else if (elapsedNanos >= timeoutNanos) {
remainingNanos = 0;
} else {
remainingNanos = timeoutNanos - elapsedNanos;
}
}
}
private void resumeAdmission() {
boolean dispatchMissed;
synchronized (this) {
if (!paused) {
return;
}
paused = false;
dispatchMissed = missedWhilePaused && !cancelled;
missedWhilePaused = false;
}
if (dispatchMissed) {
dispatch();
}
}
}
}
@@ -78,29 +78,26 @@ public class AlarmInhibitReduce implements DisposableBean {
*/
private final long sourceAlertTtl;
private final ScheduledExecutorService cleanupScheduler;
private final AlertInhibitDao alertInhibitDao;
private final VirtualThreadProperties virtualThreadProperties;
private ScheduledExecutorService cleanupScheduler;
private final ExecutorService cleanupExecutor;
private ExecutorService cleanupExecutor;
private final ScheduledDispatchTask cleanupTask;
private ScheduledDispatchTask cleanupTask;
public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao
, AlerterProperties alerterProperties) {
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true);
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults());
}
@Autowired
public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) {
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true);
}
AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties,
boolean autoStart) {
this.alarmSilenceReduce = alarmSilenceReduce;
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.alertInhibitDao = alertInhibitDao;
this.virtualThreadProperties = virtualThreadProperties == null
? VirtualThreadProperties.defaults() : virtualThreadProperties;
if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) {
this.sourceAlertTtl = alerterProperties.getInhibit().getTtl();
} else {
@@ -108,34 +105,49 @@ public class AlarmInhibitReduce implements DisposableBean {
}
inhibitRules = new ConcurrentHashMap<>(8);
sourceAlertCache = new ConcurrentHashMap<>(8);
this.cleanupScheduler = createCleanupScheduler();
this.cleanupExecutor = createCleanupExecutor(properties);
this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache);
List<AlertInhibit> inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue();
refreshInhibitRules(inhibits);
if (autoStart) {
startScheduledCleanupCache();
}
}
private void startScheduledCleanupCache() {
cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL,
synchronized void start() {
if (cleanupScheduler != null) {
return;
}
cleanupScheduler = createCleanupScheduler();
cleanupExecutor = createCleanupExecutor(virtualThreadProperties);
ScheduledDispatchTask currentCleanupTask =
new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache);
cleanupTask = currentCleanupTask;
refreshInhibitRules(alertInhibitDao.findAlertInhibitsByEnableIsTrue());
startScheduledCleanupCache(currentCleanupTask);
}
private void startScheduledCleanupCache(ScheduledDispatchTask currentCleanupTask) {
cleanupScheduler.scheduleAtFixedRate(currentCleanupTask::dispatch, CHECK_INTERVAL, CHECK_INTERVAL,
TimeUnit.MILLISECONDS);
}
void dispatchCleanupCache() {
cleanupTask.dispatch();
synchronized void dispatchCleanupCache() {
if (cleanupTask != null) {
cleanupTask.dispatch();
}
}
void beforeCleanupCacheRun() {
}
@Override
public void destroy() {
cleanupScheduler.shutdownNow();
public synchronized void destroy() {
if (cleanupTask != null) {
cleanupTask.cancel();
}
if (cleanupScheduler != null) {
cleanupScheduler.shutdownNow();
cleanupScheduler = null;
}
if (cleanupExecutor != null) {
cleanupExecutor.shutdownNow();
cleanupExecutor = null;
}
cleanupTask = null;
}
private ScheduledExecutorService createCleanupScheduler() {
@@ -191,15 +203,14 @@ public class AlarmInhibitReduce implements DisposableBean {
* If alert is inhibited, it will not be forwarded
* @param groupAlert Grouped and pending alerts to be processed
*/
public void inhibitAlarm(GroupAlert groupAlert) {
public boolean inhibitAlarm(GroupAlert groupAlert) {
if (groupAlert == null) {
log.warn("Received null GroupAlert. Skipping processing.");
return;
return false;
}
try {
if (inhibitRules.isEmpty()) {
alarmSilenceReduce.silenceAlarm(groupAlert);
return;
return alarmSilenceReduce.silenceAlarm(groupAlert);
}
// Process each individual alert
@@ -216,10 +227,12 @@ public class AlarmInhibitReduce implements DisposableBean {
// Continue processing if there are remaining alerts
if (!groupAlert.getAlerts().isEmpty()) {
alarmSilenceReduce.silenceAlarm(groupAlert);
return alarmSilenceReduce.silenceAlarm(groupAlert);
}
return true;
} catch (Exception e) {
log.error("Error inhibiting alarm for {}", groupAlert, e);
log.error("Alarm inhibit metadata processing failed");
return false;
}
}
@@ -389,6 +402,8 @@ public class AlarmInhibitReduce implements DisposableBean {
private int pendingRuns;
private boolean cancelled;
private ScheduledDispatchTask(ExecutorService executor, Runnable task) {
this.executor = executor;
this.task = task;
@@ -397,6 +412,9 @@ public class AlarmInhibitReduce implements DisposableBean {
private void dispatch() {
boolean shouldSchedule;
synchronized (this) {
if (cancelled) {
return;
}
pendingRuns++;
shouldSchedule = !running;
if (shouldSchedule) {
@@ -427,6 +445,11 @@ public class AlarmInhibitReduce implements DisposableBean {
private void scheduleNextIfNeeded() {
boolean shouldSchedule;
synchronized (this) {
if (cancelled) {
pendingRuns = 0;
running = false;
return;
}
pendingRuns = Math.max(0, pendingRuns - 1);
shouldSchedule = pendingRuns > 0;
if (!shouldSchedule) {
@@ -436,5 +459,10 @@ public class AlarmInhibitReduce implements DisposableBean {
}
scheduleRun();
}
private synchronized void cancel() {
cancelled = true;
pendingRuns = 0;
}
}
}
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.reduce;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/** Loads reducer rules and starts cleanup/grouping workers only in normal runtime. */
@Component
@ConditionalOnNormalBusinessRuntime
public final class AlarmReduceLifecycle implements CommandLineRunner {
private final AlarmInhibitReduce inhibitReduce;
private final AlarmGroupReduce groupReduce;
public AlarmReduceLifecycle(AlarmInhibitReduce inhibitReduce, AlarmGroupReduce groupReduce) {
this.inhibitReduce = inhibitReduce;
this.groupReduce = groupReduce;
}
@Override
public void run(String... args) {
inhibitReduce.start();
groupReduce.start();
}
}
@@ -46,7 +46,7 @@ public class AlarmSilenceReduce {
* If alert matches any active silence rule, it will be silenced
* @param groupAlert The alert to be processed
*/
public void silenceAlarm(GroupAlert groupAlert) {
public boolean silenceAlarm(GroupAlert groupAlert) {
List<AlertSilence> alertSilenceList = CacheFactory.getAlertSilenceCache();
if (alertSilenceList == null) {
alertSilenceList = alertSilenceDao.findAlertSilencesByEnableTrue();
@@ -72,21 +72,21 @@ public class AlarmSilenceReduce {
continue;
}
// Alert is silenced
return;
return true;
} else if (alertSilence.getType() == 1) {
// Cyclic silence rule
int currentDayOfWeek = now.getDayOfWeek().getValue();
if (alertSilence.getDays() != null && alertSilence.getDays().contains((byte) currentDayOfWeek)
&& !checkAndSave(now, alertSilence)) {
// Alert is silenced
return;
return true;
}
}
}
}
// No matching silence rule, forward the alert
dispatcherAlarm.dispatchAlarm(groupAlert);
return dispatcherAlarm.dispatchAlarm(groupAlert);
}
/**
@@ -34,7 +34,6 @@ import org.apache.hertzbeat.alert.service.NoticeTemplateMutationException;
import org.apache.hertzbeat.alert.service.NoticeConfigService;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.Lazy;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
@@ -69,7 +68,7 @@ import java.util.stream.Collectors;
@Order(value = Ordered.HIGHEST_PRECEDENCE)
@Transactional(rollbackFor = Exception.class)
@Slf4j
public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLineRunner {
public class NoticeConfigServiceImpl implements NoticeConfigService {
private static final Map<Byte, NoticeTemplate> PRESET_TEMPLATE = new HashMap<>(16);
@@ -372,8 +371,7 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
CacheFactory.clearNoticeCache();
}
@Override
public void run(String... args) throws Exception {
void loadPresetTemplates() {
try {
log.info("load default notice template in internal jar");
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.alert.service.impl;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
/** Loads preset templates only after the business runtime is opened. */
@Component
@Order(Ordered.HIGHEST_PRECEDENCE)
@ConditionalOnNormalBusinessRuntime
public final class NoticeTemplateInitializer implements CommandLineRunner {
private final NoticeConfigServiceImpl noticeConfigService;
public NoticeTemplateInitializer(NoticeConfigServiceImpl noticeConfigService) {
this.noticeConfigService = noticeConfigService;
}
@Override
public void run(String... args) {
noticeConfigService.loadPresetTemplates();
}
}
@@ -20,14 +20,18 @@ package org.apache.hertzbeat.alert.calculate.periodic;
import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC;
import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.clearInvocations;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
@@ -64,12 +68,13 @@ class PeriodicAlertRuleSchedulerTest {
void setUp() {
scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, traceCalculator, alertDefineDao,
VirtualThreadProperties.defaults());
scheduler.start();
}
@AfterEach
void tearDown() {
if (scheduler != null) {
scheduler.destroy();
scheduler.stop();
}
}
@@ -151,9 +156,10 @@ class PeriodicAlertRuleSchedulerTest {
@Test
void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException {
scheduler.destroy();
scheduler.stop();
scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, traceCalculator, alertDefineDao,
periodicProperties(1));
scheduler.start();
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
@@ -191,7 +197,9 @@ class PeriodicAlertRuleSchedulerTest {
}
@Test
void runLoadsPeriodicTraceRulesAtStartup() throws Exception {
void startLoadsPeriodicTraceRulesAtStartup() {
scheduler.stop();
clearInvocations(alertDefineDao);
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC))
.thenReturn(java.util.List.of());
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(
@@ -200,7 +208,7 @@ class PeriodicAlertRuleSchedulerTest {
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(TRACE_ALERT_THRESHOLD_TYPE_PERIODIC))
.thenReturn(java.util.List.of(traceRule(6L)));
scheduler.run();
scheduler.start();
verify(alertDefineDao).findAlertDefinesByTypeAndEnableTrue(TRACE_ALERT_THRESHOLD_TYPE_PERIODIC);
}
@@ -218,6 +226,110 @@ class PeriodicAlertRuleSchedulerTest {
assertTrue(latch.await(5, TimeUnit.SECONDS));
}
@Test
void stopDuringPendingExecutionIsIdempotentAndDoesNotResubmit() throws InterruptedException {
CountDownLatch started = new CountDownLatch(1);
CountDownLatch interrupted = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
doAnswer(invocation -> {
int current = invocations.incrementAndGet();
if (current == 1) {
started.countDown();
try {
Thread.sleep(5000L);
} catch (InterruptedException e) {
interrupted.countDown();
Thread.currentThread().interrupt();
}
} else {
secondStarted.countDown();
}
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
scheduler.updateSchedule(metricRule(8L));
assertTrue(started.await(5, TimeUnit.SECONDS));
Thread.sleep(1200L);
scheduler.stop();
scheduler.stop();
assertTrue(interrupted.await(5, TimeUnit.SECONDS));
assertFalse(secondStarted.await(1500, TimeUnit.MILLISECONDS));
assertEquals(1, invocations.get());
}
@Test
void pauseDrainsAnEnteredPeriodicCalculationWithoutStoppingScheduler() throws Exception {
CountDownLatch started = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
CountDownLatch resumed = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
doAnswer(invocation -> {
if (invocations.incrementAndGet() == 1) {
started.countDown();
release.await();
} else {
resumed.countDown();
}
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
scheduler.updateSchedule(metricRule(9L));
assertTrue(started.await(5, TimeUnit.SECONDS));
scheduler.pauseAdmission();
assertThrows(TimeoutException.class, () -> scheduler.awaitDrained(0));
release.countDown();
scheduler.awaitDrained(TimeUnit.SECONDS.toNanos(1));
scheduler.resumeAdmission();
scheduler.updateSchedule(metricRule(10L));
assertTrue(resumed.await(5, TimeUnit.SECONDS));
}
@Test
void pausedTicksCoalescePerRuleAndResumeOnceWithVirtualExecutor() throws Exception {
assertPausedTicksResumeOnce(periodicProperties(true, 1));
}
@Test
void pausedTicksCoalescePerRuleAndResumeOnceWithScheduledExecutor() throws Exception {
assertPausedTicksResumeOnce(periodicProperties(false, 1));
}
private void assertPausedTicksResumeOnce(VirtualThreadProperties properties) throws Exception {
scheduler.stop();
CountDownLatch pausedTicks = new CountDownLatch(2);
scheduler = new PeriodicAlertRuleScheduler(
metricsCalculator, logCalculator, traceCalculator, alertDefineDao, properties) {
@Override
void beforeRuleTrigger(AlertDefine rule) {
pausedTicks.countDown();
}
};
scheduler.start();
CountDownLatch calculated = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
doAnswer(invocation -> {
invocations.incrementAndGet();
calculated.countDown();
return null;
}).when(metricsCalculator).calculate(any(AlertDefine.class));
AlertDefine rule = metricRule(11L);
scheduler.pauseAdmission();
scheduler.updateSchedule(rule);
assertTrue(pausedTicks.await(3, TimeUnit.SECONDS));
scheduler.awaitDrained(0);
scheduler.resumeAdmission();
scheduler.resumeAdmission();
assertTrue(calculated.await(1, TimeUnit.SECONDS));
scheduler.cancelSchedule(rule.getId());
assertEquals(1, invocations.get());
}
private AlertDefine metricRule(Long id) {
return AlertDefine.builder()
.id(id)
@@ -239,8 +351,12 @@ class PeriodicAlertRuleSchedulerTest {
}
private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) {
return periodicProperties(true, maxConcurrentJobs);
}
private VirtualThreadProperties periodicProperties(boolean enabled, int maxConcurrentJobs) {
return new VirtualThreadProperties(
true,
enabled,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
@@ -23,7 +23,9 @@ import org.apache.hertzbeat.alert.calculate.AlarmCacheManager;
import org.apache.hertzbeat.alert.calculate.JexlExprCalculator;
import org.apache.hertzbeat.alert.dao.SingleAlertDao;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce;
import org.apache.hertzbeat.alert.service.AlertDefineService;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
@@ -40,8 +42,12 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -71,6 +77,62 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
private MetricsRealTimeAlertCalculator metricsRealTimeAlertCalculator;
@Test
void positiveCapacityMaintenanceBufferKeepsTelemetryLoopForwardingLaterSamples() throws Exception {
int sampleCount = 8;
CountDownLatch stored = new CountDownLatch(sampleCount);
InMemoryCommonDataQueue queue = new InMemoryCommonDataQueue() {
@Override
public void sendMetricsDataToStorage(CollectRep.MetricsData metricsData) {
super.sendMetricsDataToStorage(metricsData);
stored.countDown();
}
};
AlarmCommonReduce reduce = new AlarmCommonReduce(
org.mockito.Mockito.mock(AlarmGroupReduce.class), positiveReduceCapacityProperties());
AlerterWorkerPool loopPool = new AlerterWorkerPool();
MetricsRealTimeAlertCalculator calculator = new MetricsRealTimeAlertCalculator(
loopPool, queue, alertDefineService, singleAlertDao, reduce, alarmCacheManager,
new JexlExprCalculator(), false) {
@Override
protected void calculate(CollectRep.MetricsData metricsData) {
reduce.reduceAndSendAlarm(org.apache.hertzbeat.common.entity.alerter.SingleAlert.builder()
.labels(Map.of("sample", Long.toString(metricsData.getId())))
.build());
}
};
reduce.pauseAdmission();
calculator.startCalculate();
for (int index = 0; index < sampleCount; index++) {
queue.sendMetricsData(CollectRep.MetricsData.newBuilder().setId(index + 1L).build());
}
assertTrue(stored.await(2, TimeUnit.SECONDS));
for (int index = 0; index < sampleCount; index++) {
assertNotNull(queue.pollMetricsDataToStorage());
}
reduce.destroy();
loopPool.destroy();
}
private static VirtualThreadProperties positiveReduceCapacityProperties() {
return new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
10,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
new VirtualThreadProperties.QueueProperties(1, 1),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
4),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
}
@BeforeEach
public void setUp() {
MockitoAnnotations.openMocks(this);
@@ -169,6 +231,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
verify(alarmCacheManager, times(1)).getPending(any(), any());
verify(alarmCacheManager, times(1)).putFiring(any(), any(), any());
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
verify(dataQueue, times(1)).sendMetricsDataToStorage(metricsData);
}
@Test
@@ -24,6 +24,8 @@ import static org.mockito.ArgumentMatchers.anyByte;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -180,4 +182,18 @@ class AlertNoticeDispatchTest {
verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert));
verify(emitterManager).broadcast(any(String.class));
}
@Test
void postStoreFailureDoesNotChangeMetadataSuccessOutcome() {
when(alertStoreHandler.store(alert)).thenReturn(alert);
when(noticeConfigService.getReceiverFilterRule(alert))
.thenThrow(new IllegalStateException("notice unavailable"));
doThrow(new IllegalStateException("broadcast unavailable"))
.when(emitterManager).broadcast(any(String.class));
assertTrue(alertNoticeDispatch.dispatchAlarm(alert));
verify(alertStoreHandler, times(1)).store(alert);
verify(emitterManager).broadcast(any(String.class));
}
}
@@ -17,6 +17,7 @@
package org.apache.hertzbeat.alert.notice.impl;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
@@ -46,6 +47,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.ResourceBundle;
import java.util.concurrent.atomic.AtomicReference;
import org.springframework.test.util.ReflectionTestUtils;
/**
* Test case for Email Alert Notify
@@ -106,7 +109,7 @@ class EmailAlertNotifyHandlerImplTest {
.content(JsonUtil.toJson(mailServerConfig))
.build();
when(generalConfigDao.findByType(any())).thenReturn(generalConfig);
when(mailSender.getJavaMailProperties()).thenReturn(new Properties());
lenient().when(mailSender.getJavaMailProperties()).thenReturn(new Properties());
}
@Test
@@ -130,4 +133,30 @@ class EmailAlertNotifyHandlerImplTest {
assertThrows(AlertNoticeException.class,
() -> emailAlertNotifyHandler.send(receiver, template, groupAlert));
}
@Test
void configuredFromAddressIsUsedByProductionHandler() throws Exception {
AtomicReference<MimeMessage> sent = new AtomicReference<>();
JavaMailSenderImpl sender = new JavaMailSenderImpl() {
@Override
public void send(MimeMessage mimeMessage) {
sent.set(mimeMessage);
}
};
when(generalConfigDao.findByType(any())).thenReturn(null);
EmailAlertNotifyHandlerImpl handler = new EmailAlertNotifyHandlerImpl(sender, generalConfigDao);
ReflectionTestUtils.setField(handler, "host", "smtp.example.test");
ReflectionTestUtils.setField(handler, "port", 465);
ReflectionTestUtils.setField(handler, "username", "smtp-user@example.test");
ReflectionTestUtils.setField(handler, "password", "password");
ReflectionTestUtils.setField(handler, "fromAddress", "alerts@example.test");
ReflectionTestUtils.setField(handler, "sslEnable", true);
ReflectionTestUtils.setField(handler, "starttlsEnable", false);
ReflectionTestUtils.setField(handler, "bundle", bundle);
when(bundle.getString("alerter.notify.title")).thenReturn("Alert Notification");
handler.send(receiver, template, groupAlert);
assertEquals("alerts@example.test", sent.get().getFrom()[0].toString());
}
}
@@ -18,20 +18,26 @@
package org.apache.hertzbeat.alert.reduce;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -145,4 +151,208 @@ class AlarmCommonReduceTest {
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
}
@Test
void pauseDefersNewReducersAndReplaysEachOnceAfterDrain() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch resumedStarted = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
doAnswer(invocation -> {
int current = invocations.incrementAndGet();
if (current == 1) {
firstStarted.countDown();
releaseFirst.await();
} else {
resumedStarted.countDown();
}
return null;
}).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class));
alarmCommonReduce.reduceAndSendAlarm(testAlert);
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmCommonReduce.pauseAdmission();
assertThrows(TimeoutException.class, () -> alarmCommonReduce.awaitDrained(0));
alarmCommonReduce.reduceAndSendAlarm(testAlert);
releaseFirst.countDown();
alarmCommonReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class));
alarmCommonReduce.resumeAdmission();
assertTrue(resumedStarted.await(5, TimeUnit.SECONDS));
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
}
@Test
void pauseCannotLetDeferredWaiterOvertakeReservedSubmission() throws Exception {
CountDownLatch permitReserved = new CountDownLatch(1);
CountDownLatch releaseSubmission = new CountDownLatch(1);
alarmCommonReduce.destroy();
alarmCommonReduce = new TestAlarmCommonReduce(
alarmGroupReduce, permitReserved, releaseSubmission);
CountDownLatch firstRunning = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondRunning = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
doAnswer(invocation -> {
if (invocations.incrementAndGet() == 1) {
firstRunning.countDown();
releaseFirst.await();
} else {
secondRunning.countDown();
}
return null;
}).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class));
Thread admitted = Thread.ofPlatform().start(() -> alarmCommonReduce.reduceAndSendAlarm(testAlert));
assertTrue(permitReserved.await(1, TimeUnit.SECONDS));
Thread pause = Thread.ofPlatform().start(alarmCommonReduce::pauseAdmission);
while (!alarmCommonReduce.hasQueuedMaintenanceThread(pause)) {
Thread.onSpinWait();
}
Thread deferred = Thread.ofPlatform().start(() -> alarmCommonReduce.reduceAndSendAlarm(testAlert));
releaseSubmission.countDown();
admitted.join(1_000);
pause.join(1_000);
deferred.join(1_000);
assertTrue(firstRunning.await(1, TimeUnit.SECONDS));
assertThrows(TimeoutException.class, () -> alarmCommonReduce.awaitDrained(0));
releaseFirst.countDown();
alarmCommonReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class));
alarmCommonReduce.resumeAdmission();
assertTrue(secondRunning.await(1, TimeUnit.SECONDS));
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
}
@Test
void maintenanceDeferralNeverBlocksProducerAtWorkerQueueCapacity() throws Exception {
ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class);
org.mockito.Mockito.doAnswer(invocation -> {
((Runnable) invocation.getArgument(0)).run();
return null;
}).when(executor).execute(any(Runnable.class));
alarmCommonReduce.destroy();
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor);
alarmCommonReduce.pauseAdmission();
alarmCommonReduce.reduceAndSendAlarm(testAlert);
CountDownLatch callerContinued = new CountDownLatch(1);
Thread second = Thread.ofPlatform().start(() -> {
alarmCommonReduce.reduceAndSendAlarm(testAlert);
callerContinued.countDown();
});
assertTrue(callerContinued.await(1, TimeUnit.SECONDS));
second.join(1_000);
verify(executor, times(0)).execute(any(Runnable.class));
assertEquals(2, alarmCommonReduce.deferredTaskCount());
alarmCommonReduce.resumeAdmission();
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
}
@Test
void replayQueueRejectionRetainsEveryUnsubmittedDeferredTask() {
ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class);
AtomicInteger submissions = new AtomicInteger();
org.mockito.Mockito.doAnswer(invocation -> {
if (submissions.incrementAndGet() == 2) {
throw new RejectedExecutionException();
}
((Runnable) invocation.getArgument(0)).run();
return null;
}).when(executor).execute(any(Runnable.class));
alarmCommonReduce.destroy();
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor);
alarmCommonReduce.pauseAdmission();
alarmCommonReduce.reduceAndSendAlarm(testAlert);
alarmCommonReduce.reduceAndSendAlarm(testAlert);
assertThrows(RejectedExecutionException.class, alarmCommonReduce::resumeAdmission);
assertEquals(1, alarmCommonReduce.deferredTaskCount());
org.mockito.Mockito.doAnswer(invocation -> {
((Runnable) invocation.getArgument(0)).run();
return null;
}).when(executor).execute(any(Runnable.class));
alarmCommonReduce.resumeAdmission();
assertEquals(0, alarmCommonReduce.deferredTaskCount());
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
}
@Test
void rejectedReplayRemainsDeferredForOneRetry() {
ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class);
org.mockito.Mockito.doThrow(new RejectedExecutionException()).when(executor).execute(any(Runnable.class));
alarmCommonReduce.destroy();
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor);
alarmCommonReduce.pauseAdmission();
alarmCommonReduce.reduceAndSendAlarm(testAlert);
assertThrows(RejectedExecutionException.class,
alarmCommonReduce::resumeAdmission);
assertEquals(1, alarmCommonReduce.deferredTaskCount());
org.mockito.Mockito.doAnswer(invocation -> {
((Runnable) invocation.getArgument(0)).run();
return null;
}).when(executor).execute(any(Runnable.class));
alarmCommonReduce.resumeAdmission();
assertEquals(0, alarmCommonReduce.deferredTaskCount());
verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class));
}
private static final class TestAlarmCommonReduce extends AlarmCommonReduce {
private final CountDownLatch permitReserved;
private final CountDownLatch releaseSubmission;
private final AtomicBoolean first = new AtomicBoolean(true);
private TestAlarmCommonReduce(
AlarmGroupReduce alarmGroupReduce,
CountDownLatch permitReserved,
CountDownLatch releaseSubmission) {
super(alarmGroupReduce, singleWorkerProperties());
this.permitReserved = permitReserved;
this.releaseSubmission = releaseSubmission;
}
@Override
void beforeAdmittedSubmission() {
if (first.compareAndSet(true, false)) {
permitReserved.countDown();
try {
releaseSubmission.await();
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
}
}
}
}
private static VirtualThreadProperties singleWorkerProperties() {
return singleWorkerProperties(1);
}
private static VirtualThreadProperties singleWorkerProperties(int queueCapacity) {
return new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
new VirtualThreadProperties.AlerterProperties(
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
10,
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
new VirtualThreadProperties.QueueProperties(1, queueCapacity),
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
4),
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
VirtualThreadProperties.AsyncProperties.defaults());
}
}
@@ -40,18 +40,25 @@ package org.apache.hertzbeat.alert.reduce;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.clearInvocations;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao;
@@ -69,6 +76,21 @@ import org.mockito.MockitoAnnotations;
*/
class AlarmGroupReduceTest {
@Test
void constructorIsPassiveAndLifecycleIsIdempotent() {
clearInvocations(alertGroupConvergeDao);
AlarmGroupReduce inactive = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties());
verifyNoInteractions(alertGroupConvergeDao);
inactive.start();
inactive.start();
verify(alertGroupConvergeDao, times(1)).findAlertGroupConvergesByEnableIsTrue();
inactive.destroy();
inactive.destroy();
}
@Mock
private AlarmInhibitReduce alarmInhibitReduce;
@@ -80,10 +102,12 @@ class AlarmGroupReduceTest {
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
when(alarmInhibitReduce.inhibitAlarm(any())).thenReturn(true);
when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue())
.thenReturn(Collections.emptyList());
alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), false);
new VirtualThreadProperties());
alarmGroupReduce.start();
}
@AfterEach
@@ -136,6 +160,7 @@ class AlarmGroupReduceTest {
alarmGroupReduce.destroy();
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null);
alarmGroupReduce.start();
alarmGroupReduce.dispatchCheckAndSendGroups();
@@ -143,6 +168,13 @@ class AlarmGroupReduceTest {
assertTrue(virtualThread.get());
}
@Test
void dispatchAfterDestroyIsSafeNoOp() {
alarmGroupReduce.destroy();
alarmGroupReduce.dispatchCheckAndSendGroups();
}
@Test
void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
@@ -153,6 +185,7 @@ class AlarmGroupReduceTest {
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
maxConcurrent, new AtomicInteger());
alarmGroupReduce.start();
alarmGroupReduce.dispatchCheckAndSendGroups();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
@@ -165,6 +198,125 @@ class AlarmGroupReduceTest {
assertEquals(1, maxConcurrent.get());
}
@Test
void destroyWhileCheckIsRunningDropsPendingDispatch() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
alarmGroupReduce.destroy();
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
new AtomicInteger(), invocations);
alarmGroupReduce.start();
alarmGroupReduce.dispatchCheckAndSendGroups();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmGroupReduce.dispatchCheckAndSendGroups();
alarmGroupReduce.destroy();
alarmGroupReduce.dispatchCheckAndSendGroups();
assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS));
assertEquals(1, invocations.get());
}
@Test
void pauseDrainsRunningGroupPassAndCoalescesOneMissedPass() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch resumedStarted = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
alarmGroupReduce.destroy();
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, resumedStarted,
new AtomicInteger(), invocations);
alarmGroupReduce.start();
alarmGroupReduce.dispatchCheckAndSendGroups();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmGroupReduce.pauseAdmission();
assertThrows(TimeoutException.class, () -> alarmGroupReduce.awaitDrained(0));
alarmGroupReduce.dispatchCheckAndSendGroups();
releaseFirst.countDown();
alarmGroupReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
alarmGroupReduce.resumeAdmission();
assertTrue(resumedStarted.await(5, TimeUnit.SECONDS));
assertEquals(2, invocations.get());
}
@Test
void failedGroupStoreRetainsSnapshotForOneRetry() throws Exception {
alarmGroupReduce.refreshGroupDefines(List.of(groupRule(0)));
when(alarmInhibitReduce.inhibitAlarm(any())).thenReturn(false, true);
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
dispatchAndDrain();
dispatchAndDrain();
verify(alarmInhibitReduce, times(2)).inhibitAlarm(any());
}
@Test
void repeatSkipRetainsFiringUntilResolvedSnapshotIsStored() throws Exception {
AlertGroupConverge rule = groupRule(600);
alarmGroupReduce.refreshGroupDefines(List.of(rule));
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
dispatchAndDrain();
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
dispatchAndDrain();
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "resolved"));
dispatchAndDrain();
verify(alarmInhibitReduce, times(2)).inhibitAlarm(any());
}
@Test
void concurrentInsertIsNotClearedWithSuccessfulSnapshot() throws Exception {
alarmGroupReduce.refreshGroupDefines(List.of(groupRule(0)));
AtomicBoolean inserted = new AtomicBoolean();
doAnswer(invocation -> {
if (inserted.compareAndSet(false, true)) {
alarmGroupReduce.processGroupAlert(groupAlert("fp-2", "firing"));
}
return true;
}).when(alarmInhibitReduce).inhibitAlarm(any());
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
dispatchAndDrain();
dispatchAndDrain();
dispatchAndDrain();
verify(alarmInhibitReduce, times(2)).inhibitAlarm(any());
}
private void dispatchAndDrain() throws Exception {
alarmGroupReduce.dispatchCheckAndSendGroups();
alarmGroupReduce.pauseAdmission();
alarmGroupReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
alarmGroupReduce.resumeAdmission();
}
private AlertGroupConverge groupRule(long repeatInterval) {
AlertGroupConverge rule = new AlertGroupConverge();
rule.setName("test-rule");
rule.setGroupLabels(List.of("severity"));
rule.setGroupWait(0L);
rule.setGroupInterval(0L);
rule.setRepeatInterval(repeatInterval);
return rule;
}
private SingleAlert groupAlert(String fingerprint, String status) {
return SingleAlert.builder()
.fingerprint(fingerprint)
.status(status)
.labels(createLabels("severity", "critical"))
.annotations(Map.of())
.build();
}
private Map<String, String> createLabels(String... keyValues) {
Map<String, String> labels = new HashMap<>();
for (int i = 0; i < keyValues.length; i += 2) {
@@ -196,7 +348,7 @@ class AlarmGroupReduceTest {
AtomicBoolean virtualThread, CountDownLatch firstStarted,
CountDownLatch releaseFirst, CountDownLatch secondStarted,
AtomicInteger maxConcurrent, AtomicInteger invocations) {
super(alarmInhibitReduce, alertGroupConvergeDao, properties, false);
super(alarmInhibitReduce, alertGroupConvergeDao, properties);
this.virtualThreadLatch = virtualThreadLatch;
this.virtualThread = virtualThread;
this.firstStarted = firstStarted;
@@ -41,8 +41,12 @@ package org.apache.hertzbeat.alert.reduce;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.clearInvocations;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import java.util.Collections;
@@ -73,6 +77,21 @@ import org.mockito.MockitoAnnotations;
*/
class AlarmInhibitReduceTest {
@Test
void constructorIsPassiveAndLifecycleIsIdempotent() {
clearInvocations(alertInhibitDao);
AlarmInhibitReduce inactive = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties());
verifyNoInteractions(alertInhibitDao);
inactive.start();
inactive.start();
verify(alertInhibitDao, times(1)).findAlertInhibitsByEnableIsTrue();
inactive.destroy();
inactive.destroy();
}
@Mock
private AlertInhibitDao alertInhibitDao;
@@ -96,7 +115,8 @@ class AlarmInhibitReduceTest {
when(alerterProperties.getInhibit()).thenReturn(inhibitProperties);
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), false);
new VirtualThreadProperties());
alarmInhibitReduce.start();
}
@AfterEach
@@ -214,6 +234,15 @@ class AlarmInhibitReduceTest {
verify(alarmSilenceReduce).silenceAlarm(alert);
}
@Test
void synchronousSilenceOrStoreFailureIsReportedToGroupOwner() {
GroupAlert alert = GroupAlert.builder().alerts(new ArrayList<>()).build();
doThrow(new IllegalStateException("store unavailable"))
.when(alarmSilenceReduce).silenceAlarm(alert);
assertFalse(alarmInhibitReduce.inhibitAlarm(alert));
}
@Test
void whenMultipleSourceAlerts_shouldInhibitAllMatchingTargets() {
AlertInhibit rule = AlertInhibit.builder()
@@ -306,7 +335,8 @@ class AlarmInhibitReduceTest {
when(alerterProperties.getInhibit()).thenReturn(inhibitProperties);
alarmInhibitReduce.destroy();
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), false);
new VirtualThreadProperties());
alarmInhibitReduce.start();
AlertInhibit rule = AlertInhibit.builder()
.id(1L)
@@ -347,6 +377,7 @@ class AlarmInhibitReduceTest {
alarmInhibitReduce.destroy();
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null);
alarmInhibitReduce.start();
alarmInhibitReduce.dispatchCleanupCache();
@@ -354,6 +385,13 @@ class AlarmInhibitReduceTest {
assertTrue(virtualThread.get());
}
@Test
void dispatchAfterDestroyIsSafeNoOp() {
alarmInhibitReduce.destroy();
alarmInhibitReduce.dispatchCleanupCache();
}
@Test
void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
@@ -364,6 +402,7 @@ class AlarmInhibitReduceTest {
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
maxConcurrent, new AtomicInteger());
alarmInhibitReduce.start();
alarmInhibitReduce.dispatchCleanupCache();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
@@ -376,6 +415,29 @@ class AlarmInhibitReduceTest {
assertEquals(1, maxConcurrent.get());
}
@Test
void destroyWhileCleanupIsRunningDropsPendingDispatch() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger invocations = new AtomicInteger();
alarmInhibitReduce.destroy();
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
new AtomicInteger(), invocations);
alarmInhibitReduce.start();
alarmInhibitReduce.dispatchCleanupCache();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
alarmInhibitReduce.dispatchCleanupCache();
alarmInhibitReduce.destroy();
alarmInhibitReduce.dispatchCleanupCache();
assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS));
assertEquals(1, invocations.get());
}
private GroupAlert createGroupAlert(String status, Map<String, String> labels, List<SingleAlert> alerts) {
return GroupAlert.builder()
.status(status)
@@ -424,7 +486,7 @@ class AlarmInhibitReduceTest {
CountDownLatch firstStarted, CountDownLatch releaseFirst,
CountDownLatch secondStarted, AtomicInteger maxConcurrent,
AtomicInteger invocations) {
super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties, false);
super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties);
this.virtualThreadLatch = virtualThreadLatch;
this.virtualThread = virtualThread;
this.firstStarted = firstStarted;
@@ -22,6 +22,8 @@ import org.apache.hertzbeat.collector.dispatch.CollectorRuntimeStatusProvider;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.FailureCode;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.ObservedLong;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayStatus;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayTransport;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.RuntimeTelemetry;
/**
@@ -84,10 +86,24 @@ public class OtelRuntimeStatusProvider implements CollectorRuntimeStatusProvider
diagnosticsReader.sanitize(snapshot.lastError(), properties),
failureCode,
telemetry,
sources
sources,
otlpGateway(snapshot)
);
}
private OtlpGatewayStatus otlpGateway(OtelRuntimeSnapshot snapshot) {
if (!properties.isOtlpGatewayEnabled()) {
return OtlpGatewayStatus.disabled();
}
if (!properties.isEnabled() || snapshot.state() != OtelRuntimeState.RUNNING) {
return OtlpGatewayStatus.unavailable();
}
// Gateway mode renders both OTLP receivers into the same health-checked runtime.
return OtlpGatewayStatus.available(List.of(
OtlpGatewayTransport.HTTP_PROTOBUF,
OtlpGatewayTransport.GRPC));
}
private List<ManagedOtelRuntimeStatus.ManagedOtelSourceStatus> sanitize(
List<ManagedOtelRuntimeStatus.ManagedOtelSourceStatus> sources) {
return sources.stream()
@@ -41,6 +41,7 @@ class OtelRuntimeStatusProviderTest {
properties.setEnabled(true);
properties.setConfigRevision(12);
properties.setToken("managed-intake-token");
properties.setOtlpGatewayEnabled(true);
OtelRuntimeSupervisor supervisor = mock(OtelRuntimeSupervisor.class);
when(supervisor.snapshot()).thenReturn(new OtelRuntimeSnapshot(
OtelRuntimeState.RUNNING, 42, 2, Instant.parse("2026-07-15T06:00:00Z"), ""));
@@ -72,6 +73,12 @@ class OtelRuntimeStatusProviderTest {
status.intakeCredentialState());
assertEquals(List.of(source), status.sources());
assertEquals(telemetry, status.telemetry());
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.AVAILABLE,
status.otlpGateway().state());
assertEquals(List.of(
ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF,
ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC),
status.otlpGateway().supportedTransports());
}
@Test
@@ -93,6 +100,35 @@ class OtelRuntimeStatusProviderTest {
assertEquals(ManagedOtelRuntimeStatus.IntakeCredentialState.NOT_REQUIRED,
status.intakeCredentialState());
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.DISABLED,
status.otlpGateway().state());
}
@Test
void enabledGatewayIsUnavailableUntilTheManagedRuntimeIsRunning() {
OtelRuntimeProperties properties = new OtelRuntimeProperties();
properties.setEnabled(true);
properties.setOtlpGatewayEnabled(true);
properties.setToken("managed-intake-token");
OtelRuntimeSupervisor supervisor = mock(OtelRuntimeSupervisor.class);
when(supervisor.snapshot()).thenReturn(new OtelRuntimeSnapshot(
OtelRuntimeState.STARTING, -1, 0, Instant.parse("2026-07-15T06:00:00Z"), ""));
when(supervisor.sourceStatuses()).thenReturn(List.of());
OtelRuntimeDiagnosticsReader diagnosticsReader = mock(OtelRuntimeDiagnosticsReader.class);
when(diagnosticsReader.latestFailure(properties)).thenReturn(ManagedOtelRuntimeStatus.FailureCode.NONE);
when(diagnosticsReader.sanitize("", properties)).thenReturn("");
OtelRuntimeStatusProvider provider = new OtelRuntimeStatusProvider(
properties,
supervisor,
mock(OtelRuntimeTelemetryClient.class),
diagnosticsReader,
new OtelRuntimeFailureClassifier());
ManagedOtelRuntimeStatus status = provider.status();
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.UNAVAILABLE,
status.otlpGateway().state());
assertTrue(status.otlpGateway().supportedTransports().isEmpty());
}
@Test
@@ -0,0 +1,153 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.concurrent;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* Process-local admission gate for work that must drain without owning its executor.
*/
public final class WorkAdmissionGate {
private final Object lock = new Object();
private boolean accepting = true;
private boolean stopped;
private int activeWork;
private int waitingWork;
/**
* Returns a permit for admitted work, or {@code null} while admission is paused.
*/
public Permit tryAcquire() {
synchronized (lock) {
if (!accepting) {
return null;
}
activeWork++;
return new Permit(this);
}
}
/**
* Reserves replay work while ordinary admission is still paused.
*/
public Permit reserveReplay() {
synchronized (lock) {
if (stopped) {
return null;
}
activeWork++;
return new Permit(this);
}
}
/**
* Waits for resumed admission, or returns {@code null} after terminal stop.
*/
public Permit awaitAcquire() throws InterruptedException {
synchronized (lock) {
while (!accepting && !stopped) {
waitingWork++;
try {
lock.wait();
} finally {
waitingWork--;
}
}
if (stopped) {
return null;
}
activeWork++;
return new Permit(this);
}
}
public void pauseAdmission() {
synchronized (lock) {
accepting = false;
}
}
public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException {
synchronized (lock) {
long remainingNanos = timeoutNanos;
long startedNanos = System.nanoTime();
while (activeWork > 0) {
if (remainingNanos <= 0) {
throw new TimeoutException();
}
TimeUnit.NANOSECONDS.timedWait(lock, remainingNanos);
long elapsedNanos = System.nanoTime() - startedNanos;
if (elapsedNanos <= 0) {
remainingNanos = timeoutNanos;
} else if (elapsedNanos >= timeoutNanos) {
remainingNanos = 0;
} else {
remainingNanos = timeoutNanos - elapsedNanos;
}
}
}
}
public void resumeAdmission() {
synchronized (lock) {
if (!stopped) {
accepting = true;
}
lock.notifyAll();
}
}
/**
* Permanently rejects admission and wakes work waiting for a maintenance resume.
*/
public void stop() {
synchronized (lock) {
stopped = true;
accepting = false;
lock.notifyAll();
}
}
int waitingWork() {
synchronized (lock) {
return waitingWork;
}
}
private void release() {
synchronized (lock) {
if (activeWork > 0) {
activeWork--;
lock.notifyAll();
}
}
}
/**
* Idempotent ownership token for one admitted unit of work.
*/
public static final class Permit implements AutoCloseable {
private final WorkAdmissionGate gate;
private boolean closed;
private Permit(WorkAdmissionGate gate) {
this.gate = gate;
}
@Override
public synchronized void close() {
if (!closed) {
closed = true;
gate.release();
}
}
}
}
@@ -57,21 +57,18 @@ public interface CommonConstants {
*/
byte LOGIN_FAILED_CODE = 0x05;
/**
* Monitoring status 0: Paused, 1: Up, 2: Down
*/
/** Monitoring status 0: Paused. */
byte MONITOR_PAUSED_CODE = 0x00;
/**
* Monitoring status 0: Paused, 1: Up, 2: Down
*/
/** Monitoring status 1: Up. */
byte MONITOR_UP_CODE = 0x01;
/**
* Monitoring status 0: Paused, 1: Up, 2: Down
*/
/** Monitoring status 2: Down. */
byte MONITOR_DOWN_CODE = 0x02;
/** Monitoring status 3: Scheduled and waiting for the first availability result. */
byte MONITOR_PENDING_CODE = 0x03;
/**
* scrape type static
*/
@@ -18,6 +18,7 @@
package org.apache.hertzbeat.common.entity.dto;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.regex.Pattern;
@@ -34,10 +35,12 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
IntakeCredentialState intakeCredentialState,
int restartCount, Instant changedAt, String lastError,
FailureCode failureCode, RuntimeTelemetry telemetry,
List<ManagedOtelSourceStatus> sources) {
List<ManagedOtelSourceStatus> sources,
OtlpGatewayStatus otlpGateway) {
public static final int CURRENT_SCHEMA_VERSION = 2;
public static final int CURRENT_SCHEMA_VERSION = 3;
private static final int LEGACY_SCHEMA_VERSION = 1;
private static final int OTLP_GATEWAY_SCHEMA_VERSION = 3;
private static final int MAXIMUM_DIAGNOSTIC_LENGTH = 512;
// Active sources plus both sides of one pending/rejected replacement revision.
private static final int MAXIMUM_SOURCE_STATUSES = 147;
@@ -49,7 +52,7 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
int restartCount, Instant changedAt, String lastError) {
this(schemaVersion, enabled, state, desiredRevision, activeRevision, -1,
intakeCredentialState, restartCount, changedAt, lastError, FailureCode.NONE,
RuntimeTelemetry.unavailable(false), List.of());
RuntimeTelemetry.unavailable(false), List.of(), OtlpGatewayStatus.notReported());
}
public ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, RuntimeState state,
@@ -59,7 +62,20 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
List<ManagedOtelSourceStatus> sources) {
this(schemaVersion, enabled, state, desiredRevision, activeRevision, -1,
intakeCredentialState, restartCount, changedAt, lastError, FailureCode.NONE,
RuntimeTelemetry.unavailable(hasFileSource(sources)), sources);
RuntimeTelemetry.unavailable(hasFileSource(sources)), sources,
OtlpGatewayStatus.notReported());
}
/** Preserves the schema 1/2 constructor shape while gateway reporting rolls out. */
public ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, RuntimeState state,
long desiredRevision, long activeRevision, long pid,
IntakeCredentialState intakeCredentialState,
int restartCount, Instant changedAt, String lastError,
FailureCode failureCode, RuntimeTelemetry telemetry,
List<ManagedOtelSourceStatus> sources) {
this(schemaVersion, enabled, state, desiredRevision, activeRevision, pid,
intakeCredentialState, restartCount, changedAt, lastError, failureCode,
telemetry, sources, OtlpGatewayStatus.notReported());
}
public ManagedOtelRuntimeStatus {
@@ -91,6 +107,11 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
}
failureCode = Objects.requireNonNullElse(failureCode, FailureCode.NONE);
telemetry = telemetry == null ? RuntimeTelemetry.unavailable(hasFileSource(sources)) : telemetry;
otlpGateway = otlpGateway == null ? OtlpGatewayStatus.notReported() : otlpGateway;
if (schemaVersion < OTLP_GATEWAY_SCHEMA_VERSION
&& otlpGateway.state() != OtlpGatewayState.NOT_REPORTED) {
throw new IllegalArgumentException("OTLP Gateway status requires managed runtime status schema 3");
}
}
private static boolean hasFileSource(List<ManagedOtelSourceStatus> sources) {
@@ -134,6 +155,57 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
UNKNOWN
}
/** Lifecycle of the explicitly enabled application-telemetry Gateway listener. */
public enum OtlpGatewayState {
NOT_REPORTED,
DISABLED,
UNAVAILABLE,
AVAILABLE
}
/** Payload-free OTLP protocols confirmed by the running managed runtime. */
public enum OtlpGatewayTransport {
HTTP_PROTOBUF,
GRPC
}
/** Bounded runtime proof used to validate separately advertised public endpoints. */
public record OtlpGatewayStatus(OtlpGatewayState state,
List<OtlpGatewayTransport> supportedTransports) {
public OtlpGatewayStatus {
state = Objects.requireNonNull(state, "state");
supportedTransports = supportedTransports == null
? List.of()
: supportedTransports.stream()
.distinct()
.sorted(Comparator.comparingInt(OtlpGatewayTransport::ordinal))
.toList();
if (state == OtlpGatewayState.AVAILABLE && supportedTransports.isEmpty()) {
throw new IllegalArgumentException("Available OTLP Gateway must report a transport");
}
if (state != OtlpGatewayState.AVAILABLE && !supportedTransports.isEmpty()) {
throw new IllegalArgumentException("Unavailable OTLP Gateway cannot report transports");
}
}
public static OtlpGatewayStatus notReported() {
return new OtlpGatewayStatus(OtlpGatewayState.NOT_REPORTED, List.of());
}
public static OtlpGatewayStatus disabled() {
return new OtlpGatewayStatus(OtlpGatewayState.DISABLED, List.of());
}
public static OtlpGatewayStatus unavailable() {
return new OtlpGatewayStatus(OtlpGatewayState.UNAVAILABLE, List.of());
}
public static OtlpGatewayStatus available(List<OtlpGatewayTransport> transports) {
return new OtlpGatewayStatus(OtlpGatewayState.AVAILABLE, transports);
}
}
/**
* Availability of one numeric runtime metric. Zero is meaningful only when available.
*/
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.runtime;
import java.util.Objects;
/** Represents the business-runtime boundary decided before an application context starts. */
public final class BusinessRuntimeGate {
private final RuntimeMode mode;
private BusinessRuntimeGate(RuntimeMode mode) {
this.mode = Objects.requireNonNull(mode, "mode");
}
public static BusinessRuntimeGate fixed(RuntimeMode mode) {
return new BusinessRuntimeGate(mode);
}
public RuntimeMode mode() {
return mode;
}
public boolean isOpen() {
return mode == RuntimeMode.NORMAL;
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.runtime;
import java.util.Locale;
/** Application runtime scope selected before a Spring context starts. */
public enum RuntimeMode {
SETUP_ONLY("setup_only"),
FULL_SETUP_GATED("full_setup_gated"),
NORMAL("normal"),
RECOVERY("recovery");
public static final String PROPERTY_NAME = "hertzbeat.runtime.mode";
private final String value;
RuntimeMode(String value) {
this.value = value;
}
public String value() {
return value;
}
public static RuntimeMode fromProperty(String value) {
if (value == null) {
return NORMAL;
}
String normalized = value.trim().toLowerCase(Locale.ROOT);
for (RuntimeMode mode : values()) {
if (mode.value.equals(normalized)) {
return mode;
}
}
throw new IllegalArgumentException("Unsupported runtime mode");
}
}
@@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.concurrent;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.Test;
class WorkAdmissionGateTest {
@Test
void pauseRejectsNewWorkAndDrainsOnlyAdmittedWork() throws Exception {
WorkAdmissionGate gate = new WorkAdmissionGate();
WorkAdmissionGate.Permit admitted = gate.tryAcquire();
gate.pauseAdmission();
assertThat(gate.tryAcquire()).isNull();
assertThatThrownBy(() -> gate.awaitDrained(0))
.isInstanceOf(TimeoutException.class);
admitted.close();
gate.awaitDrained(TimeUnit.SECONDS.toNanos(1));
gate.resumeAdmission();
assertThat(gate.tryAcquire()).isNotNull().satisfies(WorkAdmissionGate.Permit::close);
}
@Test
void repeatedPauseResumeDoesNotDuplicatePermits() throws Exception {
WorkAdmissionGate gate = new WorkAdmissionGate();
gate.pauseAdmission();
gate.pauseAdmission();
gate.awaitDrained(0);
gate.resumeAdmission();
gate.resumeAdmission();
WorkAdmissionGate.Permit permit = gate.tryAcquire();
assertThat(permit).isNotNull();
permit.close();
gate.awaitDrained(0);
}
@Test
void waitingAdmissionResumesOnceAndInterruptDoesNotLeakPermit() throws Exception {
WorkAdmissionGate gate = new WorkAdmissionGate();
gate.pauseAdmission();
AtomicReference<WorkAdmissionGate.Permit> resumed = new AtomicReference<>();
Thread waiter = Thread.ofPlatform().start(() -> {
try {
resumed.set(gate.awaitAcquire());
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
}
});
while (gate.waitingWork() == 0) {
Thread.onSpinWait();
}
gate.resumeAdmission();
waiter.join(1_000);
assertThat(waiter.isAlive()).isFalse();
assertThat(resumed.get()).isNotNull();
resumed.get().close();
gate.awaitDrained(0);
gate.pauseAdmission();
AtomicBoolean interrupted = new AtomicBoolean();
Thread interruptedWaiter = Thread.ofPlatform().start(() -> {
try {
gate.awaitAcquire();
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
interrupted.set(Thread.currentThread().isInterrupted());
}
});
while (gate.waitingWork() == 0) {
Thread.onSpinWait();
}
interruptedWaiter.interrupt();
interruptedWaiter.join(1_000);
assertThat(interrupted.get()).isTrue();
gate.awaitDrained(0);
}
@Test
void terminalStopWakesWaiterAndResumeCannotReviveAdmission() throws Exception {
WorkAdmissionGate gate = new WorkAdmissionGate();
gate.pauseAdmission();
AtomicReference<WorkAdmissionGate.Permit> result = new AtomicReference<>();
Thread waiter = Thread.ofPlatform().start(() -> {
try {
result.set(gate.awaitAcquire());
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
}
});
while (gate.waitingWork() == 0) {
Thread.onSpinWait();
}
gate.stop();
gate.resumeAdmission();
waiter.join(1_000);
assertThat(waiter.isAlive()).isFalse();
assertThat(result.get()).isNull();
assertThat(gate.tryAcquire()).isNull();
}
}
@@ -190,5 +190,38 @@ class ManagedOtelRuntimeStatusTest {
status.telemetry().queueSize().state());
assertEquals(ManagedOtelRuntimeStatus.ValueState.UNAVAILABLE,
status.telemetry().queueCapacityBySignal().traces().state());
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.NOT_REPORTED,
status.otlpGateway().state());
}
@Test
void carriesOnlyBoundedPayloadFreeOtlpGatewayCapabilities() {
ManagedOtelRuntimeStatus.OtlpGatewayStatus gateway =
ManagedOtelRuntimeStatus.OtlpGatewayStatus.available(List.of(
ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF,
ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC));
ManagedOtelRuntimeStatus status = new ManagedOtelRuntimeStatus(
ManagedOtelRuntimeStatus.CURRENT_SCHEMA_VERSION,
true,
ManagedOtelRuntimeStatus.RuntimeState.RUNNING,
1,
1,
-1,
ManagedOtelRuntimeStatus.IntakeCredentialState.CONFIGURED,
0,
Instant.now(),
"",
ManagedOtelRuntimeStatus.FailureCode.NONE,
ManagedOtelRuntimeStatus.RuntimeTelemetry.unavailable(false),
List.of(),
gateway);
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.AVAILABLE,
status.otlpGateway().state());
assertEquals(List.of(
ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF,
ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC),
status.otlpGateway().supportedTransports());
assertFalse(status.toString().contains("endpoint"));
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.runtime;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
class BusinessRuntimeGateTest {
@Test
void opensOnlyForNormalRuntime() {
for (RuntimeMode mode : RuntimeMode.values()) {
BusinessRuntimeGate gate = BusinessRuntimeGate.fixed(mode);
assertEquals(mode, gate.mode());
assertEquals(mode == RuntimeMode.NORMAL, gate.isOpen());
}
}
@Test
void missingModePreservesExistingNormalStartup() {
assertEquals(RuntimeMode.NORMAL, RuntimeMode.fromProperty(null));
}
}
+5
View File
@@ -68,6 +68,11 @@
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.alerter;
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
import jakarta.persistence.Convert;
@@ -72,16 +73,19 @@ public class GroupAlert {
@Schema(title = "Group Labels", example = "{\"alertname\": \"HighCPUUsage\"}")
@Convert(converter = JsonMapAttributeConverter.class)
@Column(length = 2048)
@JsonInclude(content = JsonInclude.Include.NON_NULL)
private Map<String, String> groupLabels;
@Schema(title = "Common Labels", example = "{\"alertname\": \"HighCPUUsage\", \"instance\": \"server1\", \"severity\": \"critical\"}")
@Convert(converter = JsonMapAttributeConverter.class)
@Column(length = 2048)
@JsonInclude(content = JsonInclude.Include.NON_NULL)
private Map<String, String> commonLabels;
@Schema(title = "Common Annotations", example = "{\"summary\": \"High CPU usage detected\", \"description\": \"CPU usage is back to normal for server1\"}")
@Convert(converter = JsonMapAttributeConverter.class)
@Column(columnDefinition = "TEXT")
@JsonInclude(content = JsonInclude.Include.NON_NULL)
private Map<String, String> commonAnnotations;
@Schema(title = "Alert Fingerprints", example = "[\"dxsdfdsf\"]")
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.alerter;
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonInclude;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
import jakarta.persistence.Convert;
@@ -67,11 +68,13 @@ public class SingleAlert {
@Schema(title = "Labels", example = "{\"alertname\": \"HighCPUUsage\", \"priority\": \"critical\", \"instance\": \"343483943\"}")
@Convert(converter = JsonMapAttributeConverter.class)
@Column(length = 2048)
@JsonInclude(content = JsonInclude.Include.NON_NULL)
private Map<String, String> labels;
@Schema(title = "Annotations", example = "{\"summary\": \"High CPU usage detected\"}")
@Convert(converter = JsonMapAttributeConverter.class)
@Column(length = 4096)
@JsonInclude(content = JsonInclude.Include.NON_NULL)
private Map<String, String> annotations;
@Schema(title = "Content", example = "CPU usage is above 80% for the last 5 minutes on instance server1.example.com.")
@@ -98,7 +98,7 @@ public class Monitor {
@Size(max = 100)
private String cronExpression;
@Schema(title = "Task status 0: Paused, 1: Up, 2: Down", accessMode = READ_WRITE)
@Schema(title = "Task status 0: Paused, 1: Up, 2: Down, 3: Pending", accessMode = READ_WRITE)
@Min(0)
@Max(4)
private byte status;
@@ -25,6 +25,7 @@ import java.util.List;
public interface ObservabilityAccessTokenGateway {
String CLAIM_MANAGED = "managed";
String CLAIM_CREDENTIAL_VERSION = "credentialVersion";
/**
* Check the status of a managed token.
@@ -62,9 +63,10 @@ public interface ObservabilityAccessTokenGateway {
*
* @param userId subject from token
* @param claimedRoles roles embedded in token
* @param credentialVersion account credential generation embedded in token, or null for legacy accounts
* @return null when owner is still allowed, otherwise rejection reason
*/
String checkManagedTokenAccess(String userId, List<String> claimedRoles);
String checkManagedTokenAccess(String userId, List<String> claimedRoles, Long credentialVersion);
/**
* Touch token last used time.
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.runtime;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
/** Provides the immutable runtime gate selected before this context was launched. */
@Configuration(proxyBeanMethods = false)
public class BusinessRuntimeConfiguration {
@Bean
@ConditionalOnMissingBean
public BusinessRuntimeGate businessRuntimeGate(Environment environment) {
return BusinessRuntimeGate.fixed(RuntimeMode.fromProperty(environment.getProperty(RuntimeMode.PROPERTY_NAME)));
}
}
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.runtime;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
/** Registers an active side-effect boundary only in the normal full runtime. */
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@ConditionalOnProperty(name = RuntimeMode.PROPERTY_NAME, havingValue = "normal", matchIfMissing = true)
public @interface ConditionalOnNormalBusinessRuntime {
}
@@ -0,0 +1,116 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import java.lang.reflect.Method;
import org.aopalliance.intercept.MethodInterceptor;
import org.springframework.aop.Pointcut;
import org.springframework.aop.support.AbstractPointcutAdvisor;
import org.springframework.aop.support.AopUtils;
import org.springframework.aop.support.StaticMethodMatcherPointcut;
import org.springframework.core.Ordered;
import org.springframework.data.repository.Repository;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.interceptor.TransactionAttribute;
import org.springframework.transaction.interceptor.TransactionAttributeSource;
import org.springframework.transaction.support.TransactionSynchronizationManager;
/** Admits writable transactional boundaries before Spring opens their transactions. */
public final class MetadataWriteAdmissionAdvisor extends AbstractPointcutAdvisor {
private static final int ADVISOR_ORDER = Ordered.HIGHEST_PRECEDENCE + 100;
private final TransactionAttributeSource transactionAttributes;
private final MetadataWriteAdmissionCoordinator coordinator;
private final TransactionCompletionPermitRegistry transactionPermits;
private final boolean repositoryAttributes;
private final Pointcut pointcut = new TransactionAttributePointcut();
private final MethodInterceptor advice = this::invoke;
MetadataWriteAdmissionAdvisor(
TransactionAttributeSource transactionAttributes,
MetadataWriteAdmissionCoordinator coordinator,
TransactionCompletionPermitRegistry transactionPermits,
boolean repositoryAttributes) {
this.transactionAttributes = transactionAttributes;
this.coordinator = coordinator;
this.transactionPermits = transactionPermits;
this.repositoryAttributes = repositoryAttributes;
}
@Override
public Pointcut getPointcut() {
return pointcut;
}
@Override
public MethodInterceptor getAdvice() {
return advice;
}
@Override
public int getOrder() {
return ADVISOR_ORDER;
}
private Object invoke(org.aopalliance.intercept.MethodInvocation invocation) throws Throwable {
TransactionAttribute attribute = resolveAttribute(invocation.getMethod(), invocation.getThis());
if (attribute == null || attribute.isReadOnly()) {
return invocation.proceed();
}
if (transactionPermits.hasPermit()) {
return invocation.proceed();
}
if (joinsExistingPhysicalTransaction(attribute)) {
MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction();
transactionPermits.bind(permit);
transactionPermits.beginInvocation();
try {
return invocation.proceed();
} finally {
transactionPermits.endInvocation();
}
}
try (MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction()) {
transactionPermits.beginInvocation();
try {
return invocation.proceed();
} finally {
transactionPermits.endInvocation();
}
}
}
private TransactionAttribute resolveAttribute(Method method, Object target) {
Class<?> targetClass = target == null ? method.getDeclaringClass() : AopUtils.getTargetClass(target);
return transactionAttributes.getTransactionAttribute(method, targetClass);
}
private boolean joinsExistingPhysicalTransaction(TransactionAttribute attribute) {
if (!TransactionSynchronizationManager.isActualTransactionActive()) {
return false;
}
return switch (attribute.getPropagationBehavior()) {
case TransactionDefinition.PROPAGATION_REQUIRED,
TransactionDefinition.PROPAGATION_SUPPORTS,
TransactionDefinition.PROPAGATION_MANDATORY,
TransactionDefinition.PROPAGATION_NESTED -> true;
default -> false;
};
}
private final class TransactionAttributePointcut extends StaticMethodMatcherPointcut {
@Override
public boolean matches(Method method, Class<?> targetClass) {
return (repositoryAttributes || !Repository.class.isAssignableFrom(targetClass))
&& transactionAttributes.getTransactionAttribute(method, targetClass) != null;
}
}
}
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Role;
import org.springframework.transaction.interceptor.TransactionAttributeSource;
/** Spring wiring for process-local metadata write admission. */
@Configuration(proxyBeanMethods = false)
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public class MetadataWriteAdmissionConfiguration {
@Bean
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
MetadataWriteAdmissionCoordinator metadataWriteAdmissionCoordinator() {
return new MetadataWriteAdmissionCoordinator();
}
@Bean
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
TransactionCompletionPermitRegistry transactionCompletionPermitRegistry() {
return new TransactionCompletionPermitRegistry();
}
@Bean
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
MetadataWriteAdmissionAdvisor metadataWriteAdmissionAdvisor(
TransactionAttributeSource transactionAttributeSource,
MetadataWriteAdmissionCoordinator coordinator,
TransactionCompletionPermitRegistry transactionPermits) {
return new MetadataWriteAdmissionAdvisor(
transactionAttributeSource, coordinator, transactionPermits, false);
}
@Bean
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
static SpringDataWriteAdmissionBeanPostProcessor springDataWriteAdmissionBeanPostProcessor(
MetadataWriteAdmissionCoordinator coordinator,
TransactionCompletionPermitRegistry transactionPermits) {
return new SpringDataWriteAdmissionBeanPostProcessor(coordinator, transactionPermits);
}
}
@@ -0,0 +1,151 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import java.time.Duration;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
/** Coordinates process-local writable transaction admission and maintenance drain. */
public final class MetadataWriteAdmissionCoordinator {
private final ReentrantLock lock = new ReentrantLock();
private final Condition noActiveWrites = lock.newCondition();
private MetadataWriteAdmissionPhase phase = MetadataWriteAdmissionPhase.OPEN;
private String operationId;
private long epoch;
private Object leaseToken;
private int activeWrites;
/** Drain admitted writes and enter maintenance for one operation. */
public MetadataWriteMaintenanceLease acquire(String requestedOperationId, Duration timeout) {
requireValid(requestedOperationId, timeout);
long timeoutNanos = toNanos(timeout);
lock.lock();
try {
if (phase != MetadataWriteAdmissionPhase.OPEN) {
throw MetadataWriteAdmissionException.operationConflict();
}
phase = MetadataWriteAdmissionPhase.DRAINING;
operationId = requestedOperationId;
long currentEpoch = ++epoch;
Object currentToken = new Object();
leaseToken = currentToken;
long remainingNanos = timeoutNanos;
while (activeWrites > 0) {
if (remainingNanos <= 0) {
reopen(currentEpoch, currentToken);
throw MetadataWriteAdmissionException.drainTimeout();
}
try {
remainingNanos = noActiveWrites.awaitNanos(remainingNanos);
} catch (InterruptedException exception) {
reopen(currentEpoch, currentToken);
Thread.currentThread().interrupt();
throw MetadataWriteAdmissionException.acquisitionInterrupted();
}
}
phase = MetadataWriteAdmissionPhase.ACTIVE;
return new MetadataWriteMaintenanceLease(this, requestedOperationId, currentEpoch, currentToken);
} finally {
lock.unlock();
}
}
/** Return a consistent view without exposing the lease capability. */
public MetadataWriteAdmissionSnapshot snapshot() {
lock.lock();
try {
return new MetadataWriteAdmissionSnapshot(phase, operationId, epoch, activeWrites);
} finally {
lock.unlock();
}
}
TransactionPermit admitWritableTransaction() {
lock.lock();
try {
if (phase != MetadataWriteAdmissionPhase.OPEN) {
throw MetadataWriteAdmissionException.metadataWritesPaused();
}
activeWrites++;
return new TransactionPermit(this);
} finally {
lock.unlock();
}
}
void release(String releasedOperationId, long releasedEpoch, Object releasedToken) {
lock.lock();
try {
if (phase == MetadataWriteAdmissionPhase.ACTIVE
&& epoch == releasedEpoch
&& operationId.equals(releasedOperationId)
&& leaseToken == releasedToken) {
phase = MetadataWriteAdmissionPhase.OPEN;
operationId = null;
leaseToken = null;
}
} finally {
lock.unlock();
}
}
private void releaseWritableTransaction() {
lock.lock();
try {
activeWrites--;
if (activeWrites == 0) {
noActiveWrites.signalAll();
}
} finally {
lock.unlock();
}
}
private void reopen(long failedEpoch, Object failedToken) {
if (epoch == failedEpoch && leaseToken == failedToken && phase == MetadataWriteAdmissionPhase.DRAINING) {
phase = MetadataWriteAdmissionPhase.OPEN;
operationId = null;
leaseToken = null;
}
}
private void requireValid(String requestedOperationId, Duration timeout) {
if (requestedOperationId == null || requestedOperationId.isBlank()
|| timeout == null || timeout.isNegative()) {
throw MetadataWriteAdmissionException.invalidRequest();
}
}
private long toNanos(Duration timeout) {
try {
return timeout.toNanos();
} catch (ArithmeticException exception) {
throw MetadataWriteAdmissionException.invalidRequest();
}
}
static final class TransactionPermit implements AutoCloseable {
private final MetadataWriteAdmissionCoordinator coordinator;
private boolean closed;
private TransactionPermit(MetadataWriteAdmissionCoordinator coordinator) {
this.coordinator = coordinator;
}
@Override
public void close() {
if (!closed) {
closed = true;
coordinator.releaseWritableTransaction();
}
}
}
}
@@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
/** Stable, secret-free failure classifications for metadata write admission. */
public enum MetadataWriteAdmissionErrorCode {
MAINTENANCE_ACTIVE("metadata_writes_paused"),
OPERATION_CONFLICT("operation_conflict"),
DRAIN_TIMEOUT("drain_timeout"),
ACQUISITION_INTERRUPTED("acquisition_interrupted"),
INVALID_REQUEST("invalid_request");
private final String wireCode;
MetadataWriteAdmissionErrorCode(String wireCode) {
this.wireCode = wireCode;
}
/** Return the stable, safe code exposed at typed transport boundaries. */
public String wireCode() {
return wireCode;
}
}
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
/** Safe admission failure that never exposes operation identifiers or persistence details. */
public final class MetadataWriteAdmissionException extends RuntimeException {
private static final String MAINTENANCE_MESSAGE = "Metadata writes are temporarily unavailable";
private static final String CONFLICT_MESSAGE = "Metadata maintenance operation is already active";
private static final String TIMEOUT_MESSAGE = "Metadata write drain timed out";
private static final String INTERRUPTED_MESSAGE = "Metadata write drain was interrupted";
private static final String INVALID_MESSAGE = "Metadata maintenance request is invalid";
private final MetadataWriteAdmissionErrorCode code;
private MetadataWriteAdmissionException(MetadataWriteAdmissionErrorCode code, String message) {
super(message);
this.code = code;
}
/** Return the stable machine-readable classification. */
public MetadataWriteAdmissionErrorCode code() {
return code;
}
/** Return the stable, secret-free message suitable for typed transport boundaries. */
public String safeMessage() {
return getMessage();
}
/** Create the stable rejection used by typed metadata-write callers and tests. */
public static MetadataWriteAdmissionException metadataWritesPaused() {
return new MetadataWriteAdmissionException(
MetadataWriteAdmissionErrorCode.MAINTENANCE_ACTIVE, MAINTENANCE_MESSAGE);
}
static MetadataWriteAdmissionException operationConflict() {
return new MetadataWriteAdmissionException(
MetadataWriteAdmissionErrorCode.OPERATION_CONFLICT, CONFLICT_MESSAGE);
}
static MetadataWriteAdmissionException drainTimeout() {
return new MetadataWriteAdmissionException(
MetadataWriteAdmissionErrorCode.DRAIN_TIMEOUT, TIMEOUT_MESSAGE);
}
static MetadataWriteAdmissionException acquisitionInterrupted() {
return new MetadataWriteAdmissionException(
MetadataWriteAdmissionErrorCode.ACQUISITION_INTERRUPTED, INTERRUPTED_MESSAGE);
}
static MetadataWriteAdmissionException invalidRequest() {
return new MetadataWriteAdmissionException(
MetadataWriteAdmissionErrorCode.INVALID_REQUEST, INVALID_MESSAGE);
}
}
@@ -0,0 +1,15 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
/** Metadata write admission lifecycle. */
public enum MetadataWriteAdmissionPhase {
OPEN,
DRAINING,
ACTIVE
}
@@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
/** Immutable diagnostic projection of local metadata write admission state. */
public record MetadataWriteAdmissionSnapshot(
MetadataWriteAdmissionPhase phase,
String operationId,
long epoch,
int activeWritableTransactions) {
}
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import java.util.concurrent.atomic.AtomicBoolean;
/** Capability that returns metadata write admission to OPEN when its matching epoch is released. */
public final class MetadataWriteMaintenanceLease implements AutoCloseable {
private final MetadataWriteAdmissionCoordinator coordinator;
private final String operationId;
private final long epoch;
private final Object token;
private final AtomicBoolean closed = new AtomicBoolean();
MetadataWriteMaintenanceLease(
MetadataWriteAdmissionCoordinator coordinator, String operationId, long epoch, Object token) {
this.coordinator = coordinator;
this.operationId = operationId;
this.epoch = epoch;
this.token = token;
}
@Override
public void close() {
if (closed.compareAndSet(false, true)) {
coordinator.release(operationId, epoch, token);
}
}
}
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import org.springframework.aop.Advisor;
import org.springframework.aop.framework.Advised;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanInitializationException;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.core.Ordered;
import org.springframework.data.repository.Repository;
import org.springframework.transaction.interceptor.TransactionAttributeSource;
import org.springframework.transaction.interceptor.TransactionInterceptor;
import org.springframework.transaction.interceptor.TransactionalProxy;
/** Inserts write admission into each existing Spring Data transaction proxy. */
final class SpringDataWriteAdmissionBeanPostProcessor implements BeanPostProcessor, Ordered {
private final MetadataWriteAdmissionCoordinator coordinator;
private final TransactionCompletionPermitRegistry transactionPermits;
SpringDataWriteAdmissionBeanPostProcessor(
MetadataWriteAdmissionCoordinator coordinator,
TransactionCompletionPermitRegistry transactionPermits) {
this.coordinator = coordinator;
this.transactionPermits = transactionPermits;
}
@Override
public int getOrder() {
return Ordered.LOWEST_PRECEDENCE;
}
@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
if (!(bean instanceof Repository<?, ?>)
|| !(bean instanceof TransactionalProxy)
|| !(bean instanceof Advised advised)) {
return bean;
}
if (hasAdmissionAdvisor(advised)) {
return bean;
}
if (advised.isFrozen()) {
throw new BeanInitializationException("Spring Data transaction proxy is frozen");
}
int transactionAdvisorIndex = transactionAdvisorIndex(advised);
TransactionInterceptor interceptor = (TransactionInterceptor) advised
.getAdvisors()[transactionAdvisorIndex].getAdvice();
TransactionAttributeSource attributes = interceptor.getTransactionAttributeSource();
if (attributes == null) {
throw new BeanInitializationException("Spring Data transaction attributes are unavailable");
}
advised.addAdvisor(transactionAdvisorIndex,
new MetadataWriteAdmissionAdvisor(attributes, coordinator, transactionPermits, true));
return bean;
}
private boolean hasAdmissionAdvisor(Advised advised) {
for (Advisor advisor : advised.getAdvisors()) {
if (advisor instanceof MetadataWriteAdmissionAdvisor) {
return true;
}
}
return false;
}
private int transactionAdvisorIndex(Advised advised) {
int found = -1;
for (int index = 0; index < advised.getAdvisors().length; index++) {
if (advised.getAdvisors()[index].getAdvice() instanceof TransactionInterceptor) {
if (found >= 0) {
throw new BeanInitializationException("Spring Data transaction advisor is ambiguous");
}
found = index;
}
}
if (found < 0) {
throw new BeanInitializationException("Spring Data transaction advisor is unavailable");
}
return found;
}
}
@@ -0,0 +1,80 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
/** Binds a writable admission permit to the current physical transaction completion. */
final class TransactionCompletionPermitRegistry {
private final Object resourceKey = new Object();
private final SynchronizationRegistrar registrar;
private final ThreadLocal<Boolean> invocationPermit = new ThreadLocal<>();
TransactionCompletionPermitRegistry() {
this(TransactionSynchronizationManager::registerSynchronization);
}
TransactionCompletionPermitRegistry(SynchronizationRegistrar registrar) {
this.registrar = registrar;
}
boolean hasPermit() {
return invocationPermit.get() != null || TransactionSynchronizationManager.hasResource(resourceKey);
}
void beginInvocation() {
invocationPermit.set(Boolean.TRUE);
}
void endInvocation() {
invocationPermit.remove();
}
void bind(MetadataWriteAdmissionCoordinator.TransactionPermit permit) {
if (!TransactionSynchronizationManager.isActualTransactionActive()
|| !TransactionSynchronizationManager.isSynchronizationActive()) {
permit.close();
throw new IllegalStateException("Transaction synchronization is unavailable");
}
boolean bound = false;
try {
TransactionSynchronizationManager.bindResource(resourceKey, permit);
bound = true;
registrar.register(new PermitReleaseSynchronization(permit));
} catch (RuntimeException | Error failure) {
if (bound) {
TransactionSynchronizationManager.unbindResourceIfPossible(resourceKey);
}
permit.close();
throw failure;
}
}
@FunctionalInterface
interface SynchronizationRegistrar {
void register(TransactionSynchronization synchronization);
}
private final class PermitReleaseSynchronization implements TransactionSynchronization {
private final MetadataWriteAdmissionCoordinator.TransactionPermit permit;
private PermitReleaseSynchronization(MetadataWriteAdmissionCoordinator.TransactionPermit permit) {
this.permit = permit;
}
@Override
public void afterCompletion(int status) {
TransactionSynchronizationManager.unbindResourceIfPossible(resourceKey);
permit.close();
}
}
}
@@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.entity.alerter;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.junit.jupiter.api.Test;
import tools.jackson.databind.JsonNode;
class AlertEntitySerializationTest {
@Test
void shouldOmitNullMapEntriesFromAlertApiPayloads() {
Map<String, String> labels = new HashMap<>();
labels.put("alertname", "CollectorUnavailable");
labels.put("collectorVersion", null);
SingleAlert singleAlert = SingleAlert.builder()
.labels(labels)
.annotations(new HashMap<>(labels))
.build();
GroupAlert groupAlert = GroupAlert.builder()
.groupLabels(new HashMap<>(labels))
.commonLabels(new HashMap<>(labels))
.commonAnnotations(new HashMap<>(labels))
.alerts(List.of(singleAlert))
.build();
JsonNode payload = JsonUtil.fromJson(JsonUtil.toJson(groupAlert));
assertStringMapWithoutNullEntry(payload.path("groupLabels"));
assertStringMapWithoutNullEntry(payload.path("commonLabels"));
assertStringMapWithoutNullEntry(payload.path("commonAnnotations"));
assertStringMapWithoutNullEntry(payload.path("alerts").path(0).path("labels"));
assertStringMapWithoutNullEntry(payload.path("alerts").path(0).path("annotations"));
}
private void assertStringMapWithoutNullEntry(JsonNode map) {
assertEquals("CollectorUnavailable", map.path("alertname").textValue());
assertFalse(map.has("collectorVersion"));
}
}
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.common.runtime;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
class NormalBusinessRuntimeConditionTest {
private final ApplicationContextRunner runner = new ApplicationContextRunner()
.withUserConfiguration(ConditionalConfiguration.class);
@Test
void fullSetupGatedContextStartsWithoutBusinessSideEffectBean() {
runner.withPropertyValues(RuntimeMode.PROPERTY_NAME + "=full_setup_gated").run(context -> {
assertTrue(context.isRunning());
assertFalse(context.containsBean("businessSideEffect"));
});
}
@Test
void normalContextRegistersBusinessSideEffectBean() {
runner.withPropertyValues(RuntimeMode.PROPERTY_NAME + "=normal").run(context ->
assertTrue(context.containsBean("businessSideEffect")));
}
@Test
void missingRuntimeModePreservesExistingNormalStartup() {
runner.run(context -> assertTrue(context.containsBean("businessSideEffect")));
}
@Configuration(proxyBeanMethods = false)
static class ConditionalConfiguration {
@Bean
@ConditionalOnNormalBusinessRuntime
String businessSideEffect() {
return "started";
}
}
}
@@ -0,0 +1,504 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.Id;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.aop.support.AopUtils;
import org.springframework.aop.framework.Advised;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.IllegalTransactionStateException;
import org.springframework.transaction.interceptor.BeanFactoryTransactionAttributeSourceAdvisor;
import org.springframework.transaction.interceptor.TransactionInterceptor;
import org.springframework.transaction.support.TransactionSynchronization;
import org.springframework.transaction.support.TransactionSynchronizationManager;
@ExtendWith(SpringExtension.class)
@ContextConfiguration(classes = MetadataWriteAdmissionIntegrationTest.TestConfiguration.class)
class MetadataWriteAdmissionIntegrationTest {
private final ExecutorService executor = Executors.newCachedThreadPool();
@Autowired
private MetadataWriteAdmissionCoordinator coordinator;
@Autowired
private AdmissionService service;
@Autowired
private AdmissionRepository repository;
@Autowired
private MetadataWriteAdmissionAdvisor admissionAdvisor;
@Autowired
private BeanFactoryTransactionAttributeSourceAdvisor transactionAdvisor;
@AfterEach
void shutdownExecutor() {
executor.shutdownNow();
}
@BeforeEach
void clearRows() {
repository.deleteAll();
}
@Test
void drainWaitsForAdmittedTransactionCommitAndRollbackCompletion() throws Exception {
CountDownLatch commitStarted = new CountDownLatch(1);
CountDownLatch finishCommit = new CountDownLatch(1);
CountDownLatch afterCompletionEntered = new CountDownLatch(1);
CountDownLatch finishAfterCompletion = new CountDownLatch(1);
Future<?> write = executor.submit(() -> service.holdWrite(
commitStarted, finishCommit, afterCompletionEntered, finishAfterCompletion, false));
assertThat(commitStarted.await(2, TimeUnit.SECONDS)).isTrue();
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
() -> coordinator.acquire("operation-commit", Duration.ofSeconds(3)));
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
assertThat(acquiring.isDone()).isFalse();
finishCommit.countDown();
assertThat(afterCompletionEntered.await(2, TimeUnit.SECONDS)).isTrue();
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
assertThat(acquiring.isDone()).isFalse();
finishAfterCompletion.countDown();
write.get(2, TimeUnit.SECONDS);
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.ACTIVE);
}
assertThat(service.count()).isEqualTo(1);
CountDownLatch rollbackStarted = new CountDownLatch(1);
CountDownLatch finishRollback = new CountDownLatch(1);
CountDownLatch rollbackCompletionEntered = new CountDownLatch(1);
CountDownLatch finishRollbackCompletion = new CountDownLatch(1);
Future<?> rollback = executor.submit(() -> service.holdWrite(
rollbackStarted, finishRollback, rollbackCompletionEntered, finishRollbackCompletion, true));
assertThat(rollbackStarted.await(2, TimeUnit.SECONDS)).isTrue();
Future<MetadataWriteMaintenanceLease> rollbackDrain = executor.submit(
() -> coordinator.acquire("operation-rollback", Duration.ofSeconds(3)));
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
finishRollback.countDown();
assertThat(rollbackCompletionEntered.await(2, TimeUnit.SECONDS)).isTrue();
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
assertThat(rollbackDrain.isDone()).isFalse();
finishRollbackCompletion.countDown();
assertThatThrownBy(() -> rollback.get(2, TimeUnit.SECONDS)).isInstanceOf(ExecutionException.class);
try (MetadataWriteMaintenanceLease ignored = rollbackDrain.get(2, TimeUnit.SECONDS)) {
assertThat(service.count()).isEqualTo(1);
}
}
@Test
void drainingAndActiveRejectNewWritesButAllowReadOnlyAndRepositoryTransactions() throws Exception {
CountDownLatch admitted = new CountDownLatch(1);
CountDownLatch finish = new CountDownLatch(1);
Future<?> existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false));
assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue();
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
() -> coordinator.acquire("operation-gate", Duration.ofSeconds(3)));
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
assertMaintenanceRejection(() -> service.write("draining"));
assertThat(service.count()).isZero();
finish.countDown();
existing.get(2, TimeUnit.SECONDS);
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
assertMaintenanceRejection(() -> service.write("active"));
assertMaintenanceRejection(() -> repository.save(new AdmissionRow("repository")));
assertMaintenanceRejection(() -> repository.deleteById(1L));
assertThat(repository.count()).isEqualTo(1);
assertThat(repository.findAll()).hasSize(1);
assertThat(repository.declaredReadOnly()).hasSize(1);
assertMaintenanceRejection(repository::declaredWritable);
assertMaintenanceRejection(service::readOnlyThenNestedWrite);
}
service.write("open-again");
assertThat(service.count()).isEqualTo(2);
}
@Test
void nestedWritableBoundaryReusesOuterThreadPermitWhileDrainStarts() throws Exception {
CountDownLatch outerAdmitted = new CountDownLatch(1);
CountDownLatch invokeNested = new CountDownLatch(1);
Future<?> outer = executor.submit(() -> service.outerThenNested(outerAdmitted, invokeNested));
assertThat(outerAdmitted.await(2, TimeUnit.SECONDS)).isTrue();
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
() -> coordinator.acquire("operation-nested", Duration.ofSeconds(3)));
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
invokeNested.countDown();
outer.get(2, TimeUnit.SECONDS);
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
assertThat(repository.count()).isEqualTo(2);
}
}
@Test
void requiredWriteJoinedToReadOnlyOuterTransactionKeepsPermitUntilPhysicalCompletion() throws Exception {
CountDownLatch nestedReturned = new CountDownLatch(1);
CountDownLatch finishOuter = new CountDownLatch(1);
Future<?> outer = executor.submit(() -> service.readOnlyOuterHoldingAfterNestedWrite(
nestedReturned, finishOuter));
assertThat(nestedReturned.await(2, TimeUnit.SECONDS)).isTrue();
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
() -> coordinator.acquire("operation-read-only-outer", Duration.ofSeconds(3)));
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
assertThat(acquiring.isDone()).isFalse();
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
finishOuter.countDown();
outer.get(2, TimeUnit.SECONDS);
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
assertThat(repository.count()).isEqualTo(1);
}
}
@Test
void timeoutRestoresOpenAndLeaseEpochPreventsStaleOrConcurrentRelease() throws Exception {
CountDownLatch admitted = new CountDownLatch(1);
CountDownLatch finish = new CountDownLatch(1);
Future<?> existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false));
assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue();
assertThatThrownBy(() -> coordinator.acquire("operation-timeout", Duration.ofMillis(100)))
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class,
error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.DRAIN_TIMEOUT));
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
service.write("accepted-after-timeout");
finish.countDown();
existing.get(2, TimeUnit.SECONDS);
MetadataWriteMaintenanceLease stale = coordinator.acquire("operation-one", Duration.ofSeconds(1));
assertConflict(() -> coordinator.acquire("operation-one", Duration.ofSeconds(1)));
assertConflict(() -> coordinator.acquire("operation-other", Duration.ofSeconds(1)));
stale.close();
MetadataWriteMaintenanceLease current = coordinator.acquire("operation-two", Duration.ofSeconds(1));
stale.close();
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.ACTIVE);
assertThat(coordinator.snapshot().operationId()).isEqualTo("operation-two");
current.close();
current.close();
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
}
@Test
void interruptedDrainReopensAdmissionAndTransactionStartFailureDoesNotLeakPermit() throws Exception {
assertThatThrownBy(service::mandatoryWrite)
.isInstanceOf(IllegalTransactionStateException.class);
assertThat(coordinator.snapshot().activeWritableTransactions()).isZero();
CountDownLatch admitted = new CountDownLatch(1);
CountDownLatch finish = new CountDownLatch(1);
Future<?> existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false));
assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue();
AtomicReference<MetadataWriteAdmissionException> failure = new AtomicReference<>();
CountDownLatch interrupted = new CountDownLatch(1);
Future<?> acquiring = executor.submit(() -> {
try {
coordinator.acquire("operation-interrupted", Duration.ofSeconds(30));
} catch (MetadataWriteAdmissionException exception) {
failure.set(exception);
} finally {
interrupted.countDown();
}
});
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
acquiring.cancel(true);
assertThat(interrupted.await(2, TimeUnit.SECONDS)).isTrue();
assertThat(failure.get().code()).isEqualTo(MetadataWriteAdmissionErrorCode.ACQUISITION_INTERRUPTED);
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
finish.countDown();
existing.get(2, TimeUnit.SECONDS);
}
@Test
void validatesOperationAndDurationWithoutLeakingRawFailures() {
assertInvalid(() -> coordinator.acquire(" ", Duration.ofSeconds(1)));
assertInvalid(() -> coordinator.acquire("operation", null));
assertInvalid(() -> coordinator.acquire("operation", Duration.ofSeconds(-1)));
assertInvalid(() -> coordinator.acquire("operation", Duration.ofSeconds(Long.MAX_VALUE)));
MetadataWriteMaintenanceLease lease = coordinator.acquire("zero-wait", Duration.ZERO);
lease.close();
}
@Test
void admissionAdvisorUsesTransactionAttributesAndRunsOutsideTransactionInterceptor() throws Exception {
assertThat(((Advised) service).getAdvisors()).contains(admissionAdvisor);
assertThat(admissionAdvisor.getOrder()).isLessThan(transactionAdvisor.getOrder());
assertThat(admissionAdvisor.getPointcut().getMethodMatcher().matches(
AdmissionService.class.getMethod("write", String.class), AdmissionService.class)).isTrue();
assertThat(AopUtils.isAopProxy(repository)).isTrue();
Advised repositoryProxy = (Advised) repository;
assertThat(AopUtils.isAopProxy(repositoryProxy.getTargetSource().getTarget())).isFalse();
assertThat(Arrays.stream(repositoryProxy.getAdvisors())
.filter(MetadataWriteAdmissionAdvisor.class::isInstance)).hasSize(1);
assertThat(Arrays.stream(repositoryProxy.getAdvisors())
.filter(advisor -> advisor.getAdvice() instanceof TransactionInterceptor)).hasSize(1);
MetadataWriteAdmissionSnapshot observed = service.observeAdmissionInsideTransaction();
assertThat(observed.phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
assertThat(observed.activeWritableTransactions()).isEqualTo(1);
}
private void awaitPhase(MetadataWriteAdmissionPhase phase) throws InterruptedException {
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);
while (coordinator.snapshot().phase() != phase && System.nanoTime() < deadline) {
Thread.onSpinWait();
}
assertThat(coordinator.snapshot().phase()).isEqualTo(phase);
}
private void assertMaintenanceRejection(Runnable write) {
assertThatThrownBy(write::run)
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class, error -> {
assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.MAINTENANCE_ACTIVE);
assertThat(error.getMessage()).isEqualTo("Metadata writes are temporarily unavailable");
assertThat(error.getCause()).isNull();
});
}
private void assertConflict(ThrowingAcquire acquire) {
assertThatThrownBy(acquire::run)
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class,
error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.OPERATION_CONFLICT));
}
private void assertInvalid(ThrowingAcquire acquire) {
assertThatThrownBy(acquire::run)
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class,
error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.INVALID_REQUEST));
}
@FunctionalInterface
private interface ThrowingAcquire {
void run();
}
@Configuration(proxyBeanMethods = false)
@EnableTransactionManagement(order = 200)
@EnableJpaRepositories(considerNestedRepositories = true,
basePackageClasses = MetadataWriteAdmissionIntegrationTest.class)
@Import(MetadataWriteAdmissionConfiguration.class)
static class TestConfiguration {
@Bean
DriverManagerDataSource dataSource() {
return new DriverManagerDataSource("jdbc:h2:mem:metadata-admission;DB_CLOSE_DELAY=-1", "sa", "");
}
@Bean
LocalContainerEntityManagerFactoryBean entityManagerFactory(DriverManagerDataSource dataSource) {
LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean();
factory.setDataSource(dataSource);
factory.setPackagesToScan(AdmissionRow.class.getPackageName());
factory.setJpaVendorAdapter(new HibernateJpaVendorAdapter());
factory.setJpaPropertyMap(Map.of("hibernate.hbm2ddl.auto", "create-drop"));
return factory;
}
@Bean
PlatformTransactionManager transactionManager(jakarta.persistence.EntityManagerFactory factory) {
return new JpaTransactionManager(factory);
}
@Bean
AdmissionNestedWriter nestedWriter(AdmissionRepository repository) {
return new AdmissionNestedWriter(repository);
}
@Bean
AdmissionService admissionService(
AdmissionRepository repository,
AdmissionNestedWriter nestedWriter,
MetadataWriteAdmissionCoordinator coordinator) {
return new AdmissionService(repository, nestedWriter, coordinator);
}
}
@Entity(name = "AdmissionRow")
static class AdmissionRow {
@Id
@GeneratedValue
private Long id;
private String label;
protected AdmissionRow() {
}
AdmissionRow(String value) {
this.label = value;
}
}
interface AdmissionRepository extends JpaRepository<AdmissionRow, Long> {
@Override
<S extends AdmissionRow> S save(S entity);
@Override
void deleteById(Long id);
@Query("select row from AdmissionRow row")
@Transactional(readOnly = true)
List<AdmissionRow> declaredReadOnly();
@Query("select row from AdmissionRow row")
@Transactional
List<AdmissionRow> declaredWritable();
}
static class AdmissionNestedWriter {
private final AdmissionRepository repository;
AdmissionNestedWriter(AdmissionRepository repository) {
this.repository = repository;
}
@Transactional
public void write() {
repository.saveAndFlush(new AdmissionRow("nested"));
}
}
static class AdmissionService {
private final AdmissionRepository repository;
private final AdmissionNestedWriter nestedWriter;
private final MetadataWriteAdmissionCoordinator coordinator;
AdmissionService(
AdmissionRepository repository,
AdmissionNestedWriter nestedWriter,
MetadataWriteAdmissionCoordinator coordinator) {
this.repository = repository;
this.nestedWriter = nestedWriter;
this.coordinator = coordinator;
}
@Transactional
public void write(String value) {
repository.saveAndFlush(new AdmissionRow(value));
}
@Transactional
public void holdWrite(
CountDownLatch admitted,
CountDownLatch finish,
CountDownLatch afterCompletionEntered,
CountDownLatch finishAfterCompletion,
boolean rollback) {
repository.saveAndFlush(new AdmissionRow("held"));
if (afterCompletionEntered != null) {
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCompletion(int status) {
afterCompletionEntered.countDown();
await(finishAfterCompletion);
}
});
}
admitted.countDown();
await(finish);
if (rollback) {
throw new IllegalStateException("rollback requested");
}
}
@Transactional
public void outerThenNested(CountDownLatch admitted, CountDownLatch invokeNested) {
repository.saveAndFlush(new AdmissionRow("outer"));
admitted.countDown();
await(invokeNested);
nestedWriter.write();
}
@Transactional(readOnly = true)
public long count() {
return repository.count();
}
@Transactional(readOnly = true)
public void readOnlyThenNestedWrite() {
nestedWriter.write();
}
@Transactional(readOnly = true)
public void readOnlyOuterHoldingAfterNestedWrite(
CountDownLatch nestedReturned, CountDownLatch finishOuter) {
nestedWriter.write();
nestedReturned.countDown();
await(finishOuter);
}
@Transactional(propagation = Propagation.MANDATORY)
public void mandatoryWrite() {
repository.saveAndFlush(new AdmissionRow("mandatory"));
}
@Transactional
public MetadataWriteAdmissionSnapshot observeAdmissionInsideTransaction() {
assertThat(TransactionSynchronizationManager.isActualTransactionActive()).isTrue();
return coordinator.snapshot();
}
private static void await(CountDownLatch latch) {
try {
if (!latch.await(2, TimeUnit.SECONDS)) {
throw new IllegalStateException("test latch timed out");
}
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw new IllegalStateException("test interrupted", exception);
}
}
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.common.transaction;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.springframework.transaction.support.TransactionSynchronizationManager;
class TransactionCompletionPermitRegistryTest {
@AfterEach
void clearTransactionState() {
TransactionSynchronizationManager.clear();
}
@Test
void synchronizationRegistrationFailureReleasesPermitAndResource() {
MetadataWriteAdmissionCoordinator coordinator = new MetadataWriteAdmissionCoordinator();
TransactionCompletionPermitRegistry registry = new TransactionCompletionPermitRegistry(synchronization -> {
throw new IllegalStateException("registration failed");
});
TransactionSynchronizationManager.initSynchronization();
TransactionSynchronizationManager.setActualTransactionActive(true);
MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction();
assertThatThrownBy(() -> registry.bind(permit)).isInstanceOf(IllegalStateException.class);
assertThat(registry.hasPermit()).isFalse();
assertThat(coordinator.snapshot().activeWritableTransactions()).isZero();
}
}
@@ -76,6 +76,12 @@ resourceRole:
- /api/notice/**===post===[admin,user]
- /api/notice/**===put===[admin,user]
- /api/notice/**===delete===[admin]
- /api/config/deployment===get===[admin]
- /api/config/deployment/validate===post===[admin]
- /api/config/deployment/metadata-migrations===post===[admin]
- /api/config/deployment/metadata-migrations/*===get===[admin]
- /api/config/deployment/metadata-migrations/*/activate===post===[admin]
- /api/config/deployment/metadata-migrations/*/export===post===[admin]
- /api/config/email===get===[admin,user,guest]
- /api/config/email===post===[admin]
- /api/config/sms===get===[admin,user,guest]
@@ -18,6 +18,7 @@
package org.apache.hertzbeat.grafana.config;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.grafana.service.DatasourceService;
import org.apache.hertzbeat.grafana.service.ServiceAccountService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -28,6 +29,7 @@ import org.springframework.stereotype.Component;
* grafana init
*/
@Component
@ConditionalOnNormalBusinessRuntime
@Slf4j
public class GrafanaInit implements CommandLineRunner {
@Autowired
@@ -56,24 +56,35 @@ public class LogSseManager {
private final Map<Long, SseSubscriber> emitters = new ConcurrentHashMap<>();
private final Queue<LogEntry> logQueue = new ConcurrentLinkedQueue<>();
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r, "sse-batch-scheduler");
t.setDaemon(true);
return t;
});
private final ExecutorService senderPool = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "sse-sender");
t.setDaemon(true);
return t;
});
private final AtomicLong queueSize = new AtomicLong(0);
private ScheduledExecutorService scheduler;
private ExecutorService senderPool;
public LogSseManager() {
scheduler.scheduleAtFixedRate(this::flushBatch, BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS);
synchronized void start() {
if (scheduler != null) {
return;
}
scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r, "sse-batch-scheduler");
t.setDaemon(true);
return t;
});
senderPool = Executors.newCachedThreadPool(r -> {
Thread t = new Thread(r, "sse-sender");
t.setDaemon(true);
return t;
});
ExecutorService currentSenderPool = senderPool;
scheduler.scheduleAtFixedRate(
() -> flushBatch(currentSenderPool), BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS);
}
private final AtomicLong queueSize = new AtomicLong(0);
@PreDestroy
public void shutdown() {
public synchronized void shutdown() {
if (scheduler == null) {
return;
}
scheduler.shutdown();
senderPool.shutdown();
try {
@@ -84,6 +95,8 @@ public class LogSseManager {
}
scheduler.shutdownNow();
senderPool.shutdownNow();
scheduler = null;
senderPool = null;
}
/**
@@ -117,7 +130,13 @@ public class LogSseManager {
/**
* Flush queued logs to all subscribers in batch
*/
private void flushBatch() {
synchronized void flushBatch() {
if (senderPool != null) {
flushBatch(senderPool);
}
}
private void flushBatch(ExecutorService currentSenderPool) {
try {
if (logQueue.isEmpty() || emitters.isEmpty()) {
return;
@@ -140,7 +159,7 @@ public class LogSseManager {
SseSubscriber subscriber = e.getValue();
List<LogEntry> filtered = filterLogs(batch, subscriber.filters);
if (!filtered.isEmpty()) {
senderPool.submit(() -> sendToSubscriber(clientId, subscriber.emitter, filtered));
currentSenderPool.submit(() -> sendToSubscriber(clientId, subscriber.emitter, filtered));
}
}
} catch (Exception e) {
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.log.notice;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/** Starts log SSE delivery threads only in normal runtime. */
@Component("legacyLogSseManagerLifecycle")
@ConditionalOnNormalBusinessRuntime
public final class LogSseManagerLifecycle implements CommandLineRunner {
private final LogSseManager manager;
public LogSseManagerLifecycle(LogSseManager manager) {
this.manager = manager;
}
@Override
public void run(String... args) {
manager.start();
}
}
@@ -51,6 +51,7 @@ class LogSseManagerTest {
@BeforeEach
void setUp() {
logSseManager = new LogSseManager();
logSseManager.start();
}
@AfterEach
@@ -172,6 +173,17 @@ class LogSseManagerTest {
});
}
@Test
void flushAfterShutdownIsSafeNoOp() {
logSseManager.shutdown();
logSseManager.broadcast(createLogEntry("INFO", "after shutdown"));
logSseManager.flushBatch();
logSseManager.shutdown();
assertEquals(1, logSseManager.getQueueSize());
}
/**
* Helper method to create a subscriber and inject a mock emitter for testing
*/
@@ -190,4 +202,4 @@ class LogSseManagerTest {
.body(body)
.build();
}
}
}
+9 -1
View File
@@ -90,7 +90,6 @@
<dependency>
<groupId>org.apache.hertzbeat</groupId>
<artifactId>hertzbeat-observability</artifactId>
<scope>test</scope>
</dependency>
<!-- spring -->
<dependency>
@@ -152,6 +151,11 @@
<artifactId>mysql-connector-j</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>provided</scope>
</dependency>
<!-- email -->
<dependency>
<groupId>org.springframework.boot</groupId>
@@ -172,6 +176,10 @@
<groupId>com.usthe.sureness</groupId>
<artifactId>spring-boot3-starter-sureness</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-crypto</artifactId>
</dependency>
<!-- okhttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
@@ -0,0 +1,170 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.component.sd;
import java.time.Duration;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase;
/** Admission and drain state for the single service-discovery consumer loop. */
final class ServiceDiscoveryMaintenanceGate {
private final ReentrantLock lock = new ReentrantLock();
private final Condition stateChanged = lock.newCondition();
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
private Thread pollingThread;
private boolean polling;
private boolean processing;
private boolean maintenanceWakeup;
private boolean terminal;
void beforePoll() throws InterruptedException {
lock.lockInterruptibly();
try {
while (phase != MetadataMaintenancePhase.RUNNING && !terminal) {
stateChanged.await();
}
if (terminal) {
throw new InterruptedException();
}
polling = true;
pollingThread = Thread.currentThread();
} finally {
lock.unlock();
}
}
/** Finish acquisition and promote a returned message to in-flight work. */
boolean pollCompleted(boolean messageReturned) {
lock.lock();
try {
polling = false;
pollingThread = null;
if (messageReturned) {
processing = true;
}
boolean clearMaintenanceInterrupt = maintenanceWakeup && !terminal;
maintenanceWakeup = false;
stateChanged.signalAll();
return clearMaintenanceInterrupt;
} finally {
lock.unlock();
}
}
boolean pollInterrupted() {
lock.lock();
try {
polling = false;
pollingThread = null;
boolean expected = maintenanceWakeup && !terminal;
maintenanceWakeup = false;
stateChanged.signalAll();
return expected;
} finally {
lock.unlock();
}
}
void workCompleted() {
lock.lock();
try {
processing = false;
stateChanged.signalAll();
} finally {
lock.unlock();
}
}
void quiesce(Duration timeout) {
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
try {
lock.lockInterruptibly();
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw MetadataMaintenanceException.quiesceInterrupted();
}
try {
if (phase == MetadataMaintenancePhase.QUIESCED) {
return;
}
if (phase == MetadataMaintenancePhase.RUNNING) {
phase = MetadataMaintenancePhase.QUIESCING;
if (pollingThread != null) {
maintenanceWakeup = true;
pollingThread.interrupt();
}
}
while (polling || processing) {
long remainingNanos = deadline.remainingNanos();
if (remainingNanos <= 0) {
reopen();
throw MetadataMaintenanceException.quiesceTimeout();
}
try {
stateChanged.awaitNanos(remainingNanos);
} catch (InterruptedException exception) {
reopen();
Thread.currentThread().interrupt();
throw MetadataMaintenanceException.quiesceInterrupted();
}
}
phase = MetadataMaintenancePhase.QUIESCED;
stateChanged.signalAll();
} finally {
lock.unlock();
}
}
void resume() {
lock.lock();
try {
if (terminal) {
return;
}
if (phase != MetadataMaintenancePhase.RUNNING) {
reopen();
}
} finally {
lock.unlock();
}
}
void stop() {
lock.lock();
try {
if (terminal) {
return;
}
terminal = true;
if (pollingThread != null) {
pollingThread.interrupt();
}
stateChanged.signalAll();
} finally {
lock.unlock();
}
}
MetadataMaintenancePhase phase() {
lock.lock();
try {
return phase;
} finally {
lock.unlock();
}
}
private void reopen() {
phase = MetadataMaintenancePhase.RUNNING;
stateChanged.signalAll();
}
}
@@ -18,6 +18,14 @@
package org.apache.hertzbeat.manager.component.sd;
import com.google.common.collect.Maps;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
@@ -28,31 +36,31 @@ import org.apache.hertzbeat.common.entity.manager.Param;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.common.util.BackoffUtils;
import org.apache.hertzbeat.common.util.ExponentialBackoff;
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorBindDao;
import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.ParamDao;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase;
import org.apache.hertzbeat.manager.scheduler.ManagerWorkerPool;
import org.apache.hertzbeat.manager.service.MonitorService;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.time.LocalDateTime;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Service Discovery Worker
*/
@Slf4j
@Component
public class ServiceDiscoveryWorker implements InitializingBean {
@ConditionalOnNormalBusinessRuntime
@Order(100)
public class ServiceDiscoveryWorker
implements InitializingBean, DisposableBean, MetadataMaintenanceParticipant {
private static final String FILED_HOST = "host";
private static final String FILED_PORT = "port";
@@ -63,6 +71,7 @@ public class ServiceDiscoveryWorker implements InitializingBean {
private final CollectorMonitorBindDao collectorMonitorBindDao;
private final CommonDataQueue dataQueue;
private final ManagerWorkerPool workerPool;
private final ServiceDiscoveryMaintenanceGate maintenanceGate = new ServiceDiscoveryMaintenanceGate();
public ServiceDiscoveryWorker(MonitorService monitorService, ParamDao paramDao, MonitorDao monitorDao,
MonitorBindDao monitorBindDao, CollectorMonitorBindDao collectorMonitorBindDao,
@@ -81,12 +90,65 @@ public class ServiceDiscoveryWorker implements InitializingBean {
workerPool.executeLongRunning(new SdUpdateTask());
}
@Override
public String participantId() {
return "service-discovery";
}
@Override
public void quiesce(Duration timeout) {
maintenanceGate.quiesce(timeout);
}
@Override
public void resume() {
maintenanceGate.resume();
}
MetadataMaintenancePhase maintenancePhase() {
return maintenanceGate.phase();
}
@Override
public void destroy() {
maintenanceGate.stop();
}
private class SdUpdateTask implements Runnable {
@Override
public void run() {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
while (!Thread.currentThread().isInterrupted()) {
try (final CollectRep.MetricsData metricsData = dataQueue.pollServiceDiscoveryData()) {
CollectRep.MetricsData polledData;
try {
maintenanceGate.beforePoll();
polledData = dataQueue.pollServiceDiscoveryData();
} catch (InterruptedException interruptedException) {
if (maintenanceGate.pollInterrupted()) {
Thread.interrupted();
continue;
}
Thread.currentThread().interrupt();
break;
} catch (RuntimeException exception) {
boolean clearMaintenanceInterrupt = maintenanceGate.pollCompleted(false);
if (clearMaintenanceInterrupt) {
Thread.interrupted();
}
if (exception instanceof CommonDataQueueUnknownException) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} else {
log.error(exception.getMessage(), exception);
}
continue;
}
boolean clearMaintenanceInterrupt = maintenanceGate.pollCompleted(polledData != null);
if (clearMaintenanceInterrupt) {
Thread.interrupted();
}
try (final CollectRep.MetricsData metricsData = polledData) {
if (metricsData == null) {
continue;
}
@@ -163,15 +225,16 @@ public class ServiceDiscoveryWorker implements InitializingBean {
final Set<Long> needCancelMonitorIdSet = subMonitorBindMap.values().stream()
.map(MonitorBind::getMonitorId).collect(Collectors.toSet());
monitorService.deleteMonitors(needCancelMonitorIdSet);
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
break;
} catch (CommonDataQueueUnknownException ue) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} catch (Exception exception) {
log.error(exception.getMessage(), exception);
} finally {
if (polledData != null) {
maintenanceGate.workCompleted();
}
}
}
}
@@ -47,8 +47,13 @@ import org.apache.hertzbeat.manager.dao.MonitorDao;
import org.apache.hertzbeat.manager.dao.StatusPageComponentDao;
import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao;
import org.apache.hertzbeat.manager.dao.StatusPageOrgDao;
import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.data.jpa.domain.Specification;
import org.springframework.stereotype.Component;
@@ -57,7 +62,8 @@ import org.springframework.stereotype.Component;
*/
@Component
@Slf4j
public class CalculateStatus implements DisposableBean {
@Order(200)
public class CalculateStatus implements DisposableBean, MetadataMaintenanceParticipant {
private static final int DEFAULT_CALCULATE_INTERVAL_TIME = 300;
@@ -70,61 +76,145 @@ public class CalculateStatus implements DisposableBean {
private final MonitorDao monitorDao;
private final int intervals;
private final VirtualThreadProperties virtualThreadProperties;
private final ScheduledExecutorService calculateScheduler;
private ScheduledExecutorService calculateScheduler;
private final ScheduledExecutorService combineHistoryScheduler;
private ScheduledExecutorService combineHistoryScheduler;
private final ExecutorService calculateExecutor;
private ExecutorService calculateExecutor;
private final ExecutorService combineHistoryExecutor;
private ExecutorService combineHistoryExecutor;
private final ScheduledDispatchTask calculateTask;
private PausableDispatchTask calculateTask;
private final ScheduledDispatchTask combineHistoryTask;
private PausableDispatchTask combineHistoryTask;
private boolean started;
private volatile MetadataMaintenancePhase maintenancePhase = MetadataMaintenancePhase.RUNNING;
public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao,
StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao,
MonitorDao monitorDao) {
this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao,
VirtualThreadProperties.defaults(), true);
VirtualThreadProperties.defaults());
}
@Autowired
public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao,
StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao,
MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties) {
this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao,
virtualThreadProperties, true);
}
CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao,
StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao,
MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties, boolean autoStart) {
this.statusPageOrgDao = statusPageOrgDao;
this.monitorDao = monitorDao;
this.statusPageComponentDao = statusPageComponentDao;
this.statusPageHistoryDao = statusPageHistoryDao;
intervals = statusProperties.getCalculate() == null ? DEFAULT_CALCULATE_INTERVAL_TIME : statusProperties.getCalculate().getInterval();
this.calculateScheduler = createScheduler("status-page-calculate-%d", "Status calculate has uncaughtException.");
this.combineHistoryScheduler = createScheduler("status-page-history-%d", "History combine has uncaughtException.");
this.calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-",
"Status calculate worker has uncaughtException.");
this.combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-",
"History combine worker has uncaughtException.");
this.calculateTask = new ScheduledDispatchTask(calculateExecutor, this::runCalculate);
this.combineHistoryTask = new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory);
if (autoStart) {
startCalculate();
startCombineHistory();
StatusProperties.CalculateProperties calculateProperties = statusProperties.getCalculate();
intervals = calculateProperties == null
? DEFAULT_CALCULATE_INTERVAL_TIME : calculateProperties.getInterval();
this.virtualThreadProperties = virtualThreadProperties == null
? VirtualThreadProperties.defaults() : virtualThreadProperties;
}
synchronized void start() {
if (started) {
return;
}
try {
calculateScheduler = createScheduler(
"status-page-calculate-%d", "Status calculate has uncaughtException.");
combineHistoryScheduler = createScheduler(
"status-page-history-%d", "History combine has uncaughtException.");
calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-",
"Status calculate worker has uncaughtException.");
combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-",
"History combine worker has uncaughtException.");
PausableDispatchTask currentCalculateTask =
new PausableDispatchTask(calculateExecutor, this::runCalculate);
PausableDispatchTask currentCombineHistoryTask =
new PausableDispatchTask(combineHistoryExecutor, this::runCombineHistory);
calculateTask = currentCalculateTask;
combineHistoryTask = currentCombineHistoryTask;
startCalculate(currentCalculateTask);
startCombineHistory(currentCombineHistoryTask);
started = true;
} catch (RuntimeException | Error e) {
destroy();
throw e;
}
}
private void startCalculate() {
calculateScheduler.scheduleAtFixedRate(this::dispatchCalculate, 5, intervals, TimeUnit.SECONDS);
synchronized boolean isStarted() {
return started;
}
private void startCombineHistory() {
@Override
public String participantId() {
return "status-calculation";
}
@Override
public void quiesce(Duration timeout) {
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
PausableDispatchTask currentCalculateTask;
PausableDispatchTask currentCombineHistoryTask;
synchronized (this) {
if (maintenancePhase == MetadataMaintenancePhase.QUIESCED) {
return;
}
maintenancePhase = MetadataMaintenancePhase.QUIESCING;
currentCalculateTask = calculateTask;
currentCombineHistoryTask = combineHistoryTask;
if (currentCalculateTask != null) {
currentCalculateTask.pauseAdmission();
}
if (currentCombineHistoryTask != null) {
currentCombineHistoryTask.pauseAdmission();
}
}
try {
if (currentCalculateTask != null) {
currentCalculateTask.awaitDrained(deadline);
}
if (currentCombineHistoryTask != null) {
currentCombineHistoryTask.awaitDrained(deadline);
}
maintenancePhase = MetadataMaintenancePhase.QUIESCED;
} catch (MetadataMaintenanceException exception) {
resumeTasks(currentCalculateTask, currentCombineHistoryTask);
maintenancePhase = MetadataMaintenancePhase.RUNNING;
throw exception;
}
}
@Override
public synchronized void resume() {
if (maintenancePhase == MetadataMaintenancePhase.RUNNING) {
return;
}
maintenancePhase = MetadataMaintenancePhase.RUNNING;
resumeTasks(calculateTask, combineHistoryTask);
}
MetadataMaintenancePhase maintenancePhase() {
return maintenancePhase;
}
private void resumeTasks(PausableDispatchTask first, PausableDispatchTask second) {
if (second != null) {
second.resumeAdmission();
}
if (first != null) {
first.resumeAdmission();
}
}
private void startCalculate(PausableDispatchTask currentCalculateTask) {
calculateScheduler.scheduleAtFixedRate(currentCalculateTask::dispatch, 5, intervals, TimeUnit.SECONDS);
}
private void startCombineHistory(PausableDispatchTask currentCombineHistoryTask) {
// combine history every day at 1:00 AM
LocalDateTime now = LocalDateTime.now();
LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0);
@@ -132,7 +222,7 @@ public class CalculateStatus implements DisposableBean {
nextRun = nextRun.plusDays(1);
}
long delay = Duration.between(now, nextRun).toMillis();
combineHistoryScheduler.scheduleAtFixedRate(this::dispatchCombineHistory, delay,
combineHistoryScheduler.scheduleAtFixedRate(currentCombineHistoryTask::dispatch, delay,
TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS);
}
@@ -145,23 +235,52 @@ public class CalculateStatus implements DisposableBean {
}
void dispatchCalculate() {
calculateTask.dispatch();
PausableDispatchTask currentTask;
synchronized (this) {
currentTask = calculateTask;
}
if (currentTask != null) {
currentTask.dispatch();
}
}
void dispatchCombineHistory() {
combineHistoryTask.dispatch();
PausableDispatchTask currentTask;
synchronized (this) {
currentTask = combineHistoryTask;
}
if (currentTask != null) {
currentTask.dispatch();
}
}
@Override
public void destroy() {
calculateScheduler.shutdownNow();
combineHistoryScheduler.shutdownNow();
public synchronized void destroy() {
started = false;
if (calculateTask != null) {
calculateTask.cancel();
}
if (combineHistoryTask != null) {
combineHistoryTask.cancel();
}
if (calculateScheduler != null) {
calculateScheduler.shutdownNow();
calculateScheduler = null;
}
if (combineHistoryScheduler != null) {
combineHistoryScheduler.shutdownNow();
combineHistoryScheduler = null;
}
if (calculateExecutor != null) {
calculateExecutor.shutdownNow();
calculateExecutor = null;
}
if (combineHistoryExecutor != null) {
combineHistoryExecutor.shutdownNow();
combineHistoryExecutor = null;
}
calculateTask = null;
combineHistoryTask = null;
}
private void runCalculate() {
@@ -319,70 +438,4 @@ public class CalculateStatus implements DisposableBean {
})
.factory());
}
private static final class ScheduledDispatchTask {
private final ExecutorService executorService;
private final Runnable task;
private final Object lock = new Object();
private boolean running;
private int pendingRuns;
private ScheduledDispatchTask(ExecutorService executorService, Runnable task) {
this.executorService = executorService;
this.task = task;
}
private void dispatch() {
if (executorService == null) {
task.run();
return;
}
synchronized (lock) {
if (running) {
pendingRuns++;
return;
}
running = true;
}
submit();
}
private void submit() {
boolean submitted = false;
try {
executorService.execute(() -> {
try {
task.run();
} finally {
onComplete();
}
});
submitted = true;
} finally {
if (!submitted) {
synchronized (lock) {
running = false;
pendingRuns = 0;
}
}
}
}
private void onComplete() {
boolean shouldRunAgain;
synchronized (lock) {
if (pendingRuns > 0) {
pendingRuns--;
shouldRunAgain = true;
} else {
running = false;
shouldRunAgain = false;
}
}
if (shouldRunAgain) {
submit();
}
}
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.manager.component.status;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
/** Starts status-page calculation and history consolidation only in normal runtime. */
@Component
@ConditionalOnNormalBusinessRuntime
public final class CalculateStatusLifecycle implements CommandLineRunner {
private final CalculateStatus calculateStatus;
public CalculateStatusLifecycle(CalculateStatus calculateStatus) {
this.calculateStatus = calculateStatus;
}
@Override
public void run(String... args) {
calculateStatus.start();
}
}
@@ -0,0 +1,158 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.component.status;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline;
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException;
/** Single-flight dispatch with pause, drain, and one coalesced due run. */
final class PausableDispatchTask {
private final ExecutorService executorService;
private final Runnable task;
private final Object lock = new Object();
private boolean running;
private boolean pendingRun;
private boolean paused;
private boolean missedWhilePaused;
private boolean cancelled;
PausableDispatchTask(ExecutorService executorService, Runnable task) {
this.executorService = executorService;
this.task = task;
}
void dispatch() {
synchronized (lock) {
if (cancelled) {
return;
}
if (paused) {
missedWhilePaused = true;
return;
}
if (running) {
pendingRun = true;
return;
}
running = true;
}
runOrSubmit();
}
void pauseAdmission() {
synchronized (lock) {
paused = true;
missedWhilePaused |= pendingRun;
pendingRun = false;
}
}
void awaitDrained(MaintenanceDeadline deadline) {
synchronized (lock) {
while (running) {
long remainingNanos = deadline.remainingNanos();
if (remainingNanos <= 0) {
throw MetadataMaintenanceException.quiesceTimeout();
}
try {
TimeUnit.NANOSECONDS.timedWait(lock, remainingNanos);
} catch (InterruptedException exception) {
Thread.currentThread().interrupt();
throw MetadataMaintenanceException.quiesceInterrupted();
}
}
}
}
void resumeAdmission() {
boolean shouldRun = false;
synchronized (lock) {
if (!paused) {
return;
}
paused = false;
if (missedWhilePaused) {
missedWhilePaused = false;
if (running) {
pendingRun = true;
} else if (!cancelled) {
running = true;
shouldRun = true;
}
}
lock.notifyAll();
}
if (shouldRun) {
runOrSubmit();
}
}
void cancel() {
synchronized (lock) {
cancelled = true;
paused = true;
pendingRun = false;
missedWhilePaused = false;
lock.notifyAll();
}
}
private void runOrSubmit() {
if (executorService == null) {
try {
task.run();
} finally {
onComplete();
}
return;
}
boolean submitted = false;
try {
executorService.execute(() -> {
try {
task.run();
} finally {
onComplete();
}
});
submitted = true;
} finally {
if (!submitted) {
synchronized (lock) {
running = false;
pendingRun = false;
lock.notifyAll();
}
}
}
}
private void onComplete() {
boolean shouldRunAgain;
synchronized (lock) {
if (cancelled || paused) {
running = false;
pendingRun = false;
shouldRunAgain = false;
} else if (pendingRun) {
pendingRun = false;
shouldRunAgain = true;
} else {
running = false;
shouldRunAgain = false;
}
lock.notifyAll();
}
if (shouldRunAgain) {
runOrSubmit();
}
}
}
@@ -35,9 +35,9 @@ import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.service.AccountService;
import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl;
import org.jspecify.annotations.NonNull;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
@@ -103,7 +103,7 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
}
touchTokenLastUsedTime(token);
} catch (RuntimeException e) {
log.warn("Managed token validation failed");
log.warn("Managed token validation failed ({})", e.getClass().getSimpleName());
return writeError(response, HttpStatus.SERVICE_UNAVAILABLE, TOKEN_VALIDATION_UNAVAILABLE);
}
}
@@ -135,7 +135,8 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
if (rejectReason != null) {
return rejectReason;
}
rejectReason = accountService.checkManagedTokenAccess(getCurrentUserId(subject), extractClaimedRoles(subject));
rejectReason = accountService.checkManagedTokenAccess(
getCurrentUserId(subject), extractClaimedRoles(subject), extractCredentialVersion(subject));
if (rejectReason != null) {
return rejectReason;
}
@@ -155,7 +156,7 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
if (principalMap == null) {
return false;
}
Object managed = principalMap.getPrincipal(AccountServiceImpl.CLAIM_MANAGED);
Object managed = principalMap.getPrincipal(ObservabilityAccessTokenGateway.CLAIM_MANAGED);
return managed instanceof Boolean ? (Boolean) managed : Boolean.parseBoolean(String.valueOf(managed));
}
@@ -181,6 +182,16 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
return principal == null ? null : String.valueOf(principal);
}
private Long extractCredentialVersion(SubjectSum subject) {
PrincipalMap principalMap = subject.getPrincipalMap();
if (principalMap == null) {
return null;
}
Object claimedVersion = principalMap.getPrincipal(
ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION);
return claimedVersion instanceof Number number ? number.longValue() : null;
}
private String bindManagedCollectorBoundary(HttpServletRequest request, SubjectSum subject) {
PrincipalMap principalMap = subject.getPrincipalMap();
if (principalMap == null || !AuthTokenScopes.MANAGED_COLLECTOR_AUDIENCE.equals(
@@ -22,6 +22,7 @@ import jakarta.annotation.Resource;
import java.security.SecureRandom;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.common.util.AesUtil;
import org.apache.hertzbeat.manager.pojo.dto.MuteConfig;
import org.apache.hertzbeat.manager.pojo.dto.SystemSecret;
@@ -42,6 +43,7 @@ import org.springframework.stereotype.Component;
*/
@Component
@Order(value = Ordered.HIGHEST_PRECEDENCE + 2)
@ConditionalOnNormalBusinessRuntime
public class ConfigInitializer implements SmartLifecycle {
private boolean running = false;
@@ -104,4 +104,11 @@ public interface AuthTokenDao extends JpaRepository<AuthToken, Long>, JpaSpecifi
@Transactional
@Query("UPDATE AuthToken t SET t.lastUsedTime = :lastUsedTime WHERE t.tokenHash = :tokenHash")
void updateLastUsedTime(@Param("tokenHash") String tokenHash, @Param("lastUsedTime") LocalDateTime lastUsedTime);
@Modifying
@Transactional
@Query("UPDATE AuthToken t SET t.status = 1, t.revokedBy = :revokedBy, t.revokedTime = :revokedTime "
+ "WHERE t.creator = :creator AND t.status = 0")
int revokeActiveByCreator(@Param("creator") String creator, @Param("revokedBy") String revokedBy,
@Param("revokedTime") LocalDateTime revokedTime);
}
@@ -21,12 +21,16 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_ST
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayState;
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayTransport;
import org.apache.hertzbeat.common.entity.manager.Collector;
import org.apache.hertzbeat.common.support.exception.CommonException;
import org.apache.hertzbeat.manager.dao.CollectorDao;
import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake;
import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.ErrorCode;
import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.Gateway;
import org.apache.hertzbeat.manager.scheduler.runtime.CollectorRuntimeStatusRegistry;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -37,14 +41,22 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver
private final CollectorDao collectorDao;
private final CollectorIntakeAdvertisementCodec codec;
private final CollectorRuntimeStatusRegistry runtimeStatuses;
public CollectorIntakeAdvertisementService(CollectorDao collectorDao, CollectorIntakeAdvertisementCodec codec) {
public CollectorIntakeAdvertisementService(
CollectorDao collectorDao,
CollectorIntakeAdvertisementCodec codec,
CollectorRuntimeStatusRegistry runtimeStatuses) {
this.collectorDao = collectorDao;
this.codec = codec;
this.runtimeStatuses = runtimeStatuses;
}
@Transactional(rollbackFor = Exception.class)
public CollectorInstrumentationIntake update(String collectorName, CollectorIntakeAdvertisementRequest request) {
if (request.gateway() != Gateway.COLLECTOR) {
throw new IllegalArgumentException("Collector intake advertisement must be Collector-owned");
}
Collector collector = requireCollector(collectorName);
collector.setInstrumentationIntake(codec.encode(request));
collectorDao.save(collector);
@@ -73,13 +85,35 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver
return CollectorInstrumentationIntake.unavailable(
collectorId, ErrorCode.INTAKE_ADVERTISEMENT_INVALID);
}
if (request.gateway() == Gateway.COLLECTOR && collector.getStatus() != COLLECTOR_STATUS_ONLINE) {
return CollectorInstrumentationIntake.unavailable(
collectorId, ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE);
if (request.gateway() == Gateway.COLLECTOR) {
if (collector.getStatus() != COLLECTOR_STATUS_ONLINE || !runtimeSupports(collectorId, request)) {
return CollectorInstrumentationIntake.unavailable(
collectorId, ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE);
}
}
return request.available(collectorId);
}
private boolean runtimeSupports(String collectorId, CollectorIntakeAdvertisementRequest request) {
return runtimeStatuses.current(collectorId)
.map(reported -> supports(reported.status(), request))
.orElse(false);
}
private boolean supports(ManagedOtelRuntimeStatus status, CollectorIntakeAdvertisementRequest request) {
if (!status.enabled()
|| status.state() != ManagedOtelRuntimeStatus.RuntimeState.RUNNING
|| status.otlpGateway().state() != OtlpGatewayState.AVAILABLE) {
return false;
}
return request.capabilities().stream().allMatch(capability -> switch (capability) {
case OTLP_HTTP_PROTOBUF -> status.otlpGateway().supportedTransports()
.contains(OtlpGatewayTransport.HTTP_PROTOBUF);
case OTLP_GRPC -> status.otlpGateway().supportedTransports()
.contains(OtlpGatewayTransport.GRPC);
});
}
private Collector requireCollector(String collectorName) {
return collectorDao.findCollectorByName(collectorName)
.orElseThrow(() -> new CommonException("Collector not found: " + collectorName));
@@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
import java.util.concurrent.TimeoutException;
import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler;
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
/** Sequential cut across alert producers whose downstream path writes management metadata. */
@Component
@ConditionalOnNormalBusinessRuntime
@Order(400)
public final class AlertMetadataMaintenanceParticipant implements MetadataMaintenanceParticipant {
private final PeriodicAlertRuleScheduler periodicScheduler;
private final AlarmCommonReduce commonReduce;
private final AlarmGroupReduce groupReduce;
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
private boolean periodicPaused;
private boolean commonPaused;
private boolean groupPaused;
public AlertMetadataMaintenanceParticipant(
PeriodicAlertRuleScheduler periodicScheduler,
AlarmCommonReduce commonReduce,
AlarmGroupReduce groupReduce) {
this.periodicScheduler = periodicScheduler;
this.commonReduce = commonReduce;
this.groupReduce = groupReduce;
}
@Override
public String participantId() {
return "alert-control-metadata";
}
@Override
public synchronized void quiesce(Duration timeout) {
if (phase == MetadataMaintenancePhase.QUIESCED) {
return;
}
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
phase = MetadataMaintenancePhase.QUIESCING;
try {
periodicScheduler.pauseAdmission();
periodicPaused = true;
periodicScheduler.awaitDrained(deadline.remainingNanos());
commonReduce.pauseAdmission();
commonPaused = true;
commonReduce.awaitDrained(deadline.remainingNanos());
groupReduce.pauseAdmission();
groupPaused = true;
groupReduce.awaitDrained(deadline.remainingNanos());
phase = MetadataMaintenancePhase.QUIESCED;
} catch (InterruptedException exception) {
resumePausedStages();
Thread.currentThread().interrupt();
throw MetadataMaintenanceException.quiesceInterrupted();
} catch (TimeoutException exception) {
resumePausedStages();
throw MetadataMaintenanceException.quiesceTimeout();
} catch (RuntimeException exception) {
resumePausedStages();
throw MetadataMaintenanceException.participantFailure();
}
}
@Override
public synchronized void resume() {
if (phase == MetadataMaintenancePhase.RUNNING) {
return;
}
if (!resumePausedStages()) {
throw MetadataMaintenanceException.resumeFailure();
}
}
private boolean resumePausedStages() {
boolean resumed = true;
if (groupPaused) {
try {
groupReduce.resumeAdmission();
groupPaused = false;
} catch (RuntimeException exception) {
resumed = false;
}
}
if (commonPaused) {
try {
commonReduce.resumeAdmission();
commonPaused = false;
} catch (RuntimeException exception) {
resumed = false;
}
}
if (periodicPaused) {
try {
periodicScheduler.resumeAdmission();
periodicPaused = false;
} catch (RuntimeException exception) {
resumed = false;
}
}
phase = resumed ? MetadataMaintenancePhase.RUNNING : MetadataMaintenancePhase.RECOVERY_REQUIRED;
return resumed;
}
}
@@ -0,0 +1,211 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeoutException;
import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler;
import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate;
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
/** Coalesces collector lifecycle intent while draining metadata and paired alert work. */
@Component
@ConditionalOnNormalBusinessRuntime
@Order(300)
public final class CollectorLifecycleMaintenanceParticipant implements MetadataMaintenanceParticipant {
private final Object lock = new Object();
private final CollectorJobScheduler scheduler;
private final CollectorAlertHandler alertHandler;
private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate();
private final Map<String, Transition> pendingTransitions = new HashMap<>();
private final Set<String> runningIdentities = new HashSet<>();
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
private long generation;
public CollectorLifecycleMaintenanceParticipant(
CollectorJobScheduler scheduler, CollectorAlertHandler alertHandler) {
this.scheduler = scheduler;
this.alertHandler = alertHandler;
}
@Override
public String participantId() {
return "collector-control-metadata";
}
public void collectorOnline(String identity, CollectorInfo collectorInfo, boolean submitAlert) {
submit(identity, true, collectorInfo, submitAlert);
}
public void collectorOffline(String identity, boolean submitAlert) {
submit(identity, false, null, submitAlert);
}
@Override
public void quiesce(Duration timeout) {
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
synchronized (lock) {
if (phase == MetadataMaintenancePhase.QUIESCED) {
return;
}
phase = MetadataMaintenancePhase.QUIESCING;
maintenanceGate.pauseAdmission();
}
try {
maintenanceGate.awaitDrained(deadline.remainingNanos());
synchronized (lock) {
phase = MetadataMaintenancePhase.QUIESCED;
}
} catch (InterruptedException exception) {
reopenAfterFailedQuiesce();
Thread.currentThread().interrupt();
throw MetadataMaintenanceException.quiesceInterrupted();
} catch (TimeoutException exception) {
reopenAfterFailedQuiesce();
throw MetadataMaintenanceException.quiesceTimeout();
}
}
@Override
public void resume() {
while (true) {
Transition transition;
synchronized (lock) {
transition = pendingTransitions.values().stream()
.min(Comparator.comparingLong(Transition::generation))
.orElse(null);
if (transition == null) {
maintenanceGate.resumeAdmission();
phase = MetadataMaintenancePhase.RUNNING;
return;
}
pendingTransitions.remove(transition.identity(), transition);
runningIdentities.add(transition.identity());
}
try {
execute(transition);
} catch (RuntimeException exception) {
synchronized (lock) {
pendingTransitions.putIfAbsent(transition.identity(), transition);
runningIdentities.remove(transition.identity());
}
throw exception;
} finally {
synchronized (lock) {
runningIdentities.remove(transition.identity());
}
}
}
}
private void submit(String identity, boolean online, CollectorInfo collectorInfo, boolean submitAlert) {
WorkAdmissionGate.Permit permit;
Transition transition;
synchronized (lock) {
transition = new Transition(identity, online, collectorInfo, submitAlert, ++generation);
if (runningIdentities.contains(identity)) {
pendingTransitions.put(identity, transition);
return;
}
// A failed transition is retained only until a newer observed intent supersedes it.
pendingTransitions.remove(identity);
permit = maintenanceGate.tryAcquire();
if (permit == null) {
pendingTransitions.put(identity, transition);
return;
}
runningIdentities.add(identity);
}
executeAdmitted(transition, permit);
}
private void executeAdmitted(Transition firstTransition, WorkAdmissionGate.Permit firstPermit) {
Transition transition = firstTransition;
WorkAdmissionGate.Permit permit = firstPermit;
RuntimeException firstFailure = null;
while (true) {
boolean transitionFailed = false;
try {
execute(transition);
} catch (RuntimeException exception) {
transitionFailed = true;
if (firstFailure == null) {
firstFailure = exception;
} else if (firstFailure.getSuppressed().length == 0) {
firstFailure.addSuppressed(exception);
}
} finally {
permit.close();
}
synchronized (lock) {
runningIdentities.remove(transition.identity());
Transition next = pendingTransitions.remove(transition.identity());
if (next == null) {
if (transitionFailed) {
pendingTransitions.put(transition.identity(), transition);
}
throwIfFailed(firstFailure);
return;
}
permit = maintenanceGate.tryAcquire();
if (permit == null) {
pendingTransitions.put(next.identity(), next);
throwIfFailed(firstFailure);
return;
}
runningIdentities.add(next.identity());
transition = next;
}
}
}
private void throwIfFailed(RuntimeException failure) {
if (failure != null) {
throw failure;
}
}
private void execute(Transition transition) {
if (transition.online()) {
if (transition.submitAlert()) {
alertHandler.online(transition.identity());
}
scheduler.collectorGoOnline(transition.identity(), transition.collectorInfo());
return;
}
scheduler.collectorGoOffline(transition.identity());
if (transition.submitAlert()) {
alertHandler.offline(transition.identity());
}
}
private void reopenAfterFailedQuiesce() {
synchronized (lock) {
maintenanceGate.resumeAdmission();
phase = MetadataMaintenancePhase.RUNNING;
}
}
private record Transition(
String identity,
boolean online,
CollectorInfo collectorInfo,
boolean submitAlert,
long generation) {
}
}
@@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease;
/** Releases one acquired maintenance window in strict reverse order with retryable progress. */
final class CompositeMigrationMaintenanceLease implements MigrationMaintenanceLease {
private final DeploymentSingletonLease authorityLease;
private final MigrationSourceLease sourceLease;
private final MetadataMaintenanceLease producerLease;
private final MetadataWriteMaintenanceLease writeLease;
private final Runnable reservationRelease;
private boolean writeReleased;
private boolean producerReleased;
private boolean sourceReleased;
private boolean authorityReleased;
private boolean sourceCallbackActive;
private Thread sourceCallbackOwner;
CompositeMigrationMaintenanceLease(
DeploymentSingletonLease authorityLease,
MigrationSourceLease sourceLease,
MetadataMaintenanceLease producerLease,
MetadataWriteMaintenanceLease writeLease,
Runnable reservationRelease) {
this.authorityLease = authorityLease;
this.sourceLease = sourceLease;
this.producerLease = producerLease;
this.writeLease = writeLease;
this.reservationRelease = reservationRelease;
}
@Override
public synchronized void withSourceConnection(MigrationSourceAction action) {
if (action == null) {
throw MigrationMaintenanceException.invalidRequest();
}
if (writeReleased || producerReleased || sourceReleased || authorityReleased) {
throw MigrationMaintenanceException.operationConflict();
}
if (sourceCallbackActive) {
throw MigrationMaintenanceException.operationConflict();
}
sourceCallbackActive = true;
sourceCallbackOwner = Thread.currentThread();
try {
sourceLease.withConnection(action);
} finally {
sourceCallbackOwner = null;
sourceCallbackActive = false;
}
}
@Override
public synchronized void close() {
if (sourceCallbackActive && sourceCallbackOwner == Thread.currentThread()) {
throw MigrationMaintenanceException.operationConflict();
}
try {
releaseInOrder();
} catch (MigrationMaintenanceException exception) {
throw exception;
} catch (RuntimeException exception) {
throw MigrationMaintenanceException.resumeFailure();
}
reservationRelease.run();
}
private void releaseInOrder() {
if (!writeReleased) {
writeLease.close();
writeReleased = true;
}
if (!producerReleased) {
producerLease.resume();
producerReleased = true;
}
if (!sourceReleased) {
sourceLease.close();
sourceReleased = true;
}
if (!authorityReleased) {
authorityLease.close();
authorityReleased = true;
}
}
}
@@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.Duration;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import javax.sql.DataSource;
/** Bounded JDBC connection acquisition that closes every result arriving after its deadline. */
final class DeadlineConnectionAcquirer implements AutoCloseable {
private final DataSource dataSource;
private final ThreadPoolExecutor executor;
DeadlineConnectionAcquirer(DataSource dataSource) {
this.dataSource = dataSource;
executor = new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(),
Thread.ofPlatform().daemon(true).name("migration-source-connection", 0).factory());
}
Connection acquire(Duration timeout) {
long timeoutNanos;
try {
timeoutNanos = timeout.toNanos();
} catch (ArithmeticException exception) {
throw MigrationMaintenanceException.invalidRequest();
}
Attempt attempt = new Attempt();
try {
executor.execute(() -> connect(attempt));
} catch (RejectedExecutionException exception) {
if (executor.isShutdown()) {
throw MigrationMaintenanceException.sourceUnavailable();
}
throw MigrationMaintenanceException.timeout();
}
return attempt.await(timeoutNanos);
}
private void connect(Attempt attempt) {
Connection acquired = null;
Throwable failure = null;
try {
acquired = dataSource.getConnection();
} catch (Throwable connectionFailure) {
failure = connectionFailure;
}
attempt.complete(acquired, failure);
}
@Override
public void close() {
executor.shutdownNow();
}
private static final class Attempt {
private final Object lock = new Object();
private final CountDownLatch completed = new CountDownLatch(1);
private Connection connection;
private Throwable failure;
private boolean abandoned;
private boolean finished;
private Connection await(long timeoutNanos) {
try {
if (completed.await(timeoutNanos, TimeUnit.NANOSECONDS)) {
return claimCompleted();
}
return abandonOrClaim();
} catch (InterruptedException exception) {
abandon();
Thread.currentThread().interrupt();
throw MigrationMaintenanceException.interrupted();
}
}
private void complete(Connection acquired, Throwable acquiredFailure) {
synchronized (lock) {
if (abandoned) {
closeLate(acquired);
} else {
connection = acquired;
failure = acquiredFailure;
}
finished = true;
}
completed.countDown();
}
private Connection abandonOrClaim() {
synchronized (lock) {
if (finished) {
return claimCompletedLocked();
}
abandoned = true;
}
throw MigrationMaintenanceException.timeout();
}
private void abandon() {
synchronized (lock) {
if (!finished) {
abandoned = true;
} else {
closeLate(connection);
connection = null;
}
}
}
private Connection claimCompleted() {
synchronized (lock) {
return claimCompletedLocked();
}
}
private Connection claimCompletedLocked() {
if (failure instanceof Error error) {
throw error;
}
if (failure != null || connection == null) {
throw MigrationMaintenanceException.sourceUnavailable();
}
Connection claimed = connection;
connection = null;
return claimed;
}
private static void closeLate(Connection lateConnection) {
if (lateConnection == null) {
return;
}
try {
lateConnection.close();
} catch (SQLException | RuntimeException exception) {
// A late connection never becomes a lease; its details remain private.
}
}
}
}
@@ -0,0 +1,246 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.LongSupplier;
import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator;
import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionErrorCode;
import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException;
import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/** Composes deployment, source, producer, and transaction fences without owning their state machines. */
@Component
public final class DefaultMigrationMaintenanceOrchestrator implements MigrationMaintenanceOrchestrator {
private final ReentrantLock lock = new ReentrantLock();
private final DeploymentSingletonAuthority deploymentAuthority;
private final MigrationSourceGuard sourceGuard;
private final MetadataMaintenanceCoordinator producerCoordinator;
private final MetadataWriteAdmissionCoordinator writeCoordinator;
private final LongSupplier ticker;
private String operationId;
private Object ownerToken;
private MetadataMaintenanceLease recoveryProducerLease;
private boolean recoveryProducerCoordinatorRequired;
private MigrationSourceLease recoverySourceLease;
private DeploymentSingletonLease recoveryAuthorityLease;
@Autowired
public DefaultMigrationMaintenanceOrchestrator(
DeploymentSingletonAuthority deploymentAuthority,
MigrationSourceGuard sourceGuard,
MetadataMaintenanceCoordinator producerCoordinator,
MetadataWriteAdmissionCoordinator writeCoordinator) {
this(deploymentAuthority, sourceGuard, producerCoordinator, writeCoordinator, System::nanoTime);
}
DefaultMigrationMaintenanceOrchestrator(
DeploymentSingletonAuthority deploymentAuthority,
MigrationSourceGuard sourceGuard,
MetadataMaintenanceCoordinator producerCoordinator,
MetadataWriteAdmissionCoordinator writeCoordinator,
LongSupplier ticker) {
this.deploymentAuthority = deploymentAuthority;
this.sourceGuard = sourceGuard;
this.producerCoordinator = producerCoordinator;
this.writeCoordinator = writeCoordinator;
this.ticker = ticker;
}
@Override
public MigrationMaintenanceLease acquire(String requestedOperationId, Duration timeout) {
requireRequest(requestedOperationId);
MaintenanceDeadline deadline;
try {
deadline = MaintenanceDeadline.start(timeout, ticker);
} catch (MetadataMaintenanceException exception) {
throw MigrationMaintenanceException.invalidRequest();
}
Object token = reserve(requestedOperationId);
DeploymentSingletonLease authorityLease = null;
MigrationSourceLease sourceLease = null;
MetadataMaintenanceLease producerLease = null;
try {
authorityLease = deploymentAuthority.acquire(requestedOperationId, deadline.remaining());
sourceLease = sourceGuard.fence(requestedOperationId, deadline.remaining());
producerLease = producerCoordinator.quiesce(requestedOperationId, deadline.remaining());
MetadataWriteMaintenanceLease writeLease =
writeCoordinator.acquire(requestedOperationId, deadline.remaining());
return new CompositeMigrationMaintenanceLease(
authorityLease, sourceLease, producerLease, writeLease, () -> releaseReservation(token));
} catch (Error error) {
if (cleanupFailedAcquisition(error, producerLease, sourceLease, authorityLease)) {
releaseReservation(token);
} else {
retainRecovery(producerLease, sourceLease, authorityLease);
}
throw error;
} catch (RuntimeException exception) {
MigrationMaintenanceException primary = mapFailure(exception);
if (cleanupFailedAcquisition(primary, producerLease, sourceLease, authorityLease)) {
releaseReservation(token);
} else {
retainRecovery(producerLease, sourceLease, authorityLease);
}
throw primary;
}
}
private Object reserve(String requestedOperationId) {
lock.lock();
try {
if (ownerToken != null) {
if (!requestedOperationId.equals(operationId)
|| !hasRecovery()) {
throw MigrationMaintenanceException.operationConflict();
}
recoverFailedAcquisition();
}
Object token = new Object();
ownerToken = token;
operationId = requestedOperationId;
return token;
} finally {
lock.unlock();
}
}
private boolean cleanupFailedAcquisition(
Throwable primary,
MetadataMaintenanceLease producerLease,
MigrationSourceLease sourceLease,
DeploymentSingletonLease authorityLease) {
boolean interrupted = Thread.currentThread().isInterrupted();
boolean producerReleased = producerCoordinator.snapshot().phase() == MetadataMaintenancePhase.RUNNING;
if (producerLease != null) {
producerReleased = suppressCleanup(primary, producerLease::resume);
}
boolean sourceReleased = sourceLease == null;
if (producerReleased && sourceLease != null) {
sourceReleased = suppressCleanup(primary, sourceLease::close);
}
boolean authorityReleased = authorityLease == null;
if (producerReleased && sourceReleased && authorityLease != null) {
authorityReleased = suppressCleanup(primary, authorityLease::close);
}
if (interrupted) {
Thread.currentThread().interrupt();
}
return producerReleased && sourceReleased && authorityReleased;
}
private boolean suppressCleanup(Throwable primary, Runnable cleanup) {
try {
cleanup.run();
return true;
} catch (RuntimeException exception) {
primary.addSuppressed(MigrationMaintenanceException.resumeFailure());
return false;
}
}
private void retainRecovery(
MetadataMaintenanceLease producerLease,
MigrationSourceLease sourceLease,
DeploymentSingletonLease authorityLease) {
lock.lock();
try {
recoveryProducerLease = producerLease;
recoveryProducerCoordinatorRequired = producerLease == null
&& producerCoordinator.snapshot().phase() != MetadataMaintenancePhase.RUNNING;
recoverySourceLease = sourceLease;
recoveryAuthorityLease = authorityLease;
} finally {
lock.unlock();
}
}
private void recoverFailedAcquisition() {
try {
if (recoveryProducerLease != null) {
recoveryProducerLease.resume();
recoveryProducerLease = null;
}
if (recoveryProducerCoordinatorRequired) {
producerCoordinator.recover(operationId);
recoveryProducerCoordinatorRequired = false;
}
if (recoverySourceLease != null) {
recoverySourceLease.close();
recoverySourceLease = null;
}
if (recoveryAuthorityLease != null) {
recoveryAuthorityLease.close();
recoveryAuthorityLease = null;
}
ownerToken = null;
operationId = null;
} catch (RuntimeException exception) {
throw MigrationMaintenanceException.resumeFailure();
}
}
private boolean hasRecovery() {
return recoveryProducerLease != null
|| recoveryProducerCoordinatorRequired
|| recoverySourceLease != null
|| recoveryAuthorityLease != null;
}
private MigrationMaintenanceException mapFailure(RuntimeException exception) {
if (exception instanceof MigrationMaintenanceException migrationFailure) {
return migrationFailure;
}
if (exception instanceof MetadataMaintenanceException maintenanceFailure) {
return switch (maintenanceFailure.code()) {
case INVALID_REQUEST -> MigrationMaintenanceException.invalidRequest();
case OPERATION_CONFLICT, STALE_LEASE -> MigrationMaintenanceException.operationConflict();
case QUIESCE_TIMEOUT -> MigrationMaintenanceException.timeout();
case QUIESCE_INTERRUPTED -> MigrationMaintenanceException.interrupted();
case PARTICIPANT_FAILURE, RESUME_FAILURE -> MigrationMaintenanceException.maintenanceFailure();
};
}
if (exception instanceof MetadataWriteAdmissionException writeFailure) {
MetadataWriteAdmissionErrorCode code = writeFailure.code();
return switch (code) {
case INVALID_REQUEST -> MigrationMaintenanceException.invalidRequest();
case OPERATION_CONFLICT, MAINTENANCE_ACTIVE -> MigrationMaintenanceException.operationConflict();
case DRAIN_TIMEOUT -> MigrationMaintenanceException.timeout();
case ACQUISITION_INTERRUPTED -> MigrationMaintenanceException.interrupted();
};
}
return MigrationMaintenanceException.maintenanceFailure();
}
private void releaseReservation(Object token) {
lock.lock();
try {
if (ownerToken == token) {
ownerToken = null;
operationId = null;
recoveryProducerLease = null;
recoveryProducerCoordinatorRequired = false;
recoverySourceLease = null;
recoveryAuthorityLease = null;
}
} finally {
lock.unlock();
}
}
private void requireRequest(String requestedOperationId) {
if (requestedOperationId == null || requestedOperationId.isBlank()) {
throw MigrationMaintenanceException.invalidRequest();
}
}
}
@@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
/** Proves and fences single-manager ownership for one logical deployment. */
public interface DeploymentSingletonAuthority {
DeploymentSingletonLease acquire(String operationId, Duration timeout);
}
@@ -0,0 +1,15 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Owner capability for one authoritative deployment-singleton fence. */
public interface DeploymentSingletonLease extends AutoCloseable {
@Override
void close();
}
@@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.nio.file.Path;
import java.util.Locale;
/** Pure embedded-H2 source-safety classification. */
final class EmbeddedH2SourceClassifier {
private static final String H2_PREFIX = "jdbc:h2:";
private EmbeddedH2SourceClassifier() {
}
static boolean isSafeEmbeddedSource(String productName, String jdbcUrl) {
if (productName == null || !"h2".equals(productName.trim().toLowerCase(Locale.ROOT))
|| jdbcUrl == null) {
return false;
}
String url = jdbcUrl.trim().toLowerCase(Locale.ROOT);
if (!url.startsWith(H2_PREFIX) || hasUnsafeSetting(url)) {
return false;
}
String location = url.substring(H2_PREFIX.length());
if (location.startsWith("mem:")) {
return !isRemote(location.substring("mem:".length()));
}
String fileLocation = location.startsWith("file:")
? location.substring("file:".length())
: location;
return !isRemote(fileLocation) && (location.startsWith("file:")
|| location.startsWith("./")
|| location.startsWith("../")
|| location.startsWith(".\\")
|| location.startsWith("..\\")
|| location.startsWith("~/")
|| location.startsWith("/")
|| isWindowsDrive(location));
}
static boolean matchesConfiguredSource(String configuredUrl, String actualUrl) {
String configuredLocation = sourceLocation(configuredUrl);
String actualLocation = sourceLocation(actualUrl);
if (configuredLocation == null || actualLocation == null) {
return false;
}
if (configuredLocation.startsWith("mem:") || actualLocation.startsWith("mem:")) {
return configuredLocation.equals(actualLocation);
}
try {
return localPath(configuredLocation).equals(localPath(actualLocation));
} catch (RuntimeException exception) {
return false;
}
}
private static String sourceLocation(String jdbcUrl) {
if (jdbcUrl == null || !jdbcUrl.regionMatches(true, 0, H2_PREFIX, 0, H2_PREFIX.length())) {
return null;
}
String location = jdbcUrl.substring(H2_PREFIX.length()).split(";", 2)[0];
return location.regionMatches(true, 0, "file:", 0, "file:".length())
? location.substring("file:".length()) : location;
}
private static Path localPath(String location) {
String expanded = location.startsWith("~/") || location.startsWith("~\\")
? System.getProperty("user.home") + location.substring(1) : location;
return Path.of(expanded).toAbsolutePath().normalize();
}
private static boolean isRemote(String location) {
return location.contains("://") || location.startsWith("//") || location.startsWith("\\\\");
}
private static boolean isWindowsDrive(String location) {
return location.length() >= 3
&& Character.isLetter(location.charAt(0))
&& location.charAt(1) == ':'
&& (location.charAt(2) == '\\' || location.charAt(2) == '/');
}
private static boolean hasUnsafeSetting(String url) {
for (String setting : url.split(";")) {
String normalized = setting.strip();
if (normalized.startsWith("auto_server") || normalized.equals("file_lock=no")) {
return true;
}
}
return false;
}
}
@@ -0,0 +1,126 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.SQLException;
import java.time.Duration;
import javax.sql.DataSource;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties;
import org.springframework.stereotype.Component;
/** Holds an embedded H2 connection after proving its configured local access mode is safe. */
@Component
@ConditionalOnNormalBusinessRuntime
public final class EmbeddedH2SourceGuard implements MigrationSourceGuard, DisposableBean {
private final DataSourceProperties dataSourceProperties;
private final DeadlineConnectionAcquirer connectionAcquirer;
public EmbeddedH2SourceGuard(DataSource dataSource, DataSourceProperties dataSourceProperties) {
this.dataSourceProperties = dataSourceProperties;
this.connectionAcquirer = new DeadlineConnectionAcquirer(dataSource);
}
@Override
public MigrationSourceLease fence(String operationId, Duration timeout) {
requireRequest(operationId, timeout);
String configuredUrl = dataSourceProperties.getUrl();
if (!EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", configuredUrl)) {
throw MigrationMaintenanceException.sourceUnavailable();
}
Connection connection = connectionAcquirer.acquire(timeout);
try {
DatabaseMetaData metadata = connection.getMetaData();
String productName = metadata.getDatabaseProductName();
String actualUrl = metadata.getURL();
if (!EmbeddedH2SourceClassifier.isSafeEmbeddedSource(productName, actualUrl)
|| !EmbeddedH2SourceClassifier.matchesConfiguredSource(configuredUrl, actualUrl)) {
closeRejectedConnection(connection);
throw MigrationMaintenanceException.sourceUnavailable();
}
return new ConnectionSourceLease(connection);
} catch (MigrationMaintenanceException exception) {
throw exception;
} catch (SQLException | RuntimeException exception) {
closeRejectedConnection(connection);
throw MigrationMaintenanceException.sourceUnavailable();
}
}
@Override
public void destroy() {
connectionAcquirer.close();
}
private void requireRequest(String operationId, Duration timeout) {
if (operationId == null || operationId.isBlank() || timeout == null || timeout.isNegative()) {
throw MigrationMaintenanceException.invalidRequest();
}
}
private void closeRejectedConnection(Connection connection) {
try {
connection.close();
} catch (SQLException | RuntimeException exception) {
// The stable source failure remains primary.
}
}
private static final class ConnectionSourceLease implements MigrationSourceLease {
private final Connection connection;
private boolean closed;
private boolean callbackActive;
private Thread callbackOwner;
private ConnectionSourceLease(Connection connection) {
this.connection = connection;
}
@Override
public synchronized void withConnection(MigrationSourceAction action) {
if (action == null) {
throw MigrationMaintenanceException.invalidRequest();
}
if (closed) {
throw MigrationMaintenanceException.operationConflict();
}
if (callbackActive) {
throw MigrationMaintenanceException.operationConflict();
}
callbackActive = true;
callbackOwner = Thread.currentThread();
try {
action.execute(connection);
} finally {
callbackOwner = null;
callbackActive = false;
}
}
@Override
public synchronized void close() {
if (callbackActive && callbackOwner == Thread.currentThread()) {
throw MigrationMaintenanceException.operationConflict();
}
if (closed) {
return;
}
try {
connection.close();
closed = true;
} catch (SQLException | RuntimeException exception) {
throw MigrationMaintenanceException.resumeFailure();
}
}
}
}
@@ -0,0 +1,15 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Re-reads durable installation convergence for each migration fence. */
@FunctionalInterface
public interface InstallationConvergenceVerifier {
boolean isFullyConverged();
}
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
import java.util.function.LongSupplier;
/** Shared monotonic deadline for one process-local maintenance transition. */
public final class MaintenanceDeadline {
private final long timeoutNanos;
private final long startedNanos;
private final LongSupplier ticker;
private MaintenanceDeadline(long timeoutNanos, long startedNanos, LongSupplier ticker) {
this.timeoutNanos = timeoutNanos;
this.startedNanos = startedNanos;
this.ticker = ticker;
}
public static MaintenanceDeadline start(Duration timeout) {
return start(timeout, System::nanoTime);
}
static MaintenanceDeadline start(Duration timeout, LongSupplier ticker) {
if (timeout == null || timeout.isNegative()) {
throw MetadataMaintenanceException.invalidRequest();
}
try {
long timeoutNanos = timeout.toNanos();
return new MaintenanceDeadline(timeoutNanos, ticker.getAsLong(), ticker);
} catch (ArithmeticException exception) {
throw MetadataMaintenanceException.invalidRequest();
}
}
public long remainingNanos() {
long elapsedNanos = ticker.getAsLong() - startedNanos;
if (elapsedNanos <= 0) {
return timeoutNanos;
}
if (elapsedNanos >= timeoutNanos) {
return 0;
}
return timeoutNanos - elapsedNanos;
}
public Duration remaining() {
return Duration.ofNanos(remainingNanos());
}
}
@@ -0,0 +1,234 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;
import org.springframework.stereotype.Component;
/**
* Coordinates process-local metadata producers without controlling their shared executors.
*
* <p>A future migration workflow must quiesce this coordinator before acquiring metadata write
* admission. On exit it must release write admission before resuming this lease. Keeping those
* capabilities separate prevents producer lifecycle from becoming a transaction or datasource
* switch.</p>
*/
@Component
public final class MetadataMaintenanceCoordinator {
private final ReentrantLock lock = new ReentrantLock();
private final List<MetadataMaintenanceParticipant> participants;
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
private String operationId;
private long epoch;
private Object leaseToken;
public MetadataMaintenanceCoordinator(List<MetadataMaintenanceParticipant> participants) {
this.participants = List.copyOf(participants);
validateParticipants(this.participants);
}
/** Pause producers in registration order and drain work admitted before the pause. */
public MetadataMaintenanceLease quiesce(String requestedOperationId, Duration timeout) {
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
requireOperationId(requestedOperationId);
Acquisition acquisition = beginAcquisition(requestedOperationId);
List<MetadataMaintenanceParticipant> started = new ArrayList<>(participants.size());
try {
for (MetadataMaintenanceParticipant participant : participants) {
started.add(participant);
participant.quiesce(deadline.remaining());
}
} catch (MetadataMaintenanceException exception) {
rollback(acquisition, started);
throw exception;
} catch (Error error) {
rollback(acquisition, started);
throw error;
} catch (RuntimeException exception) {
rollback(acquisition, started);
throw MetadataMaintenanceException.participantFailure();
}
return completeAcquisition(acquisition);
}
public MetadataMaintenanceSnapshot snapshot() {
lock.lock();
try {
return new MetadataMaintenanceSnapshot(phase, operationId, epoch);
} finally {
lock.unlock();
}
}
void recover(String requestedOperationId) {
requireOperationId(requestedOperationId);
lock.lock();
try {
if (phase != MetadataMaintenancePhase.RECOVERY_REQUIRED
|| !requestedOperationId.equals(operationId)) {
throw MetadataMaintenanceException.operationConflict();
}
if (!resumeAllParticipants()) {
throw MetadataMaintenanceException.resumeFailure();
}
reopen();
} finally {
lock.unlock();
}
}
void resume(String resumedOperationId, long resumedEpoch, Object resumedToken) {
lock.lock();
try {
if (!ownsResumeLease(resumedOperationId, resumedEpoch, resumedToken)) {
throw MetadataMaintenanceException.staleLease();
}
phase = MetadataMaintenancePhase.QUIESCING;
boolean failed = false;
for (int index = participants.size() - 1; index >= 0; index--) {
try {
participants.get(index).resume();
} catch (RuntimeException exception) {
failed = true;
}
}
if (failed) {
throw MetadataMaintenanceException.resumeFailure();
}
reopen();
} finally {
lock.unlock();
}
}
private Acquisition beginAcquisition(String requestedOperationId) {
lock.lock();
try {
if (phase == MetadataMaintenancePhase.RECOVERY_REQUIRED) {
if (!requestedOperationId.equals(operationId)) {
throw MetadataMaintenanceException.operationConflict();
}
if (!resumeAllParticipants()) {
throw MetadataMaintenanceException.resumeFailure();
}
reopen();
}
if (phase != MetadataMaintenancePhase.RUNNING) {
throw MetadataMaintenanceException.operationConflict();
}
phase = MetadataMaintenancePhase.QUIESCING;
operationId = requestedOperationId;
long requestedEpoch = ++epoch;
Object requestedToken = new Object();
leaseToken = requestedToken;
return new Acquisition(requestedOperationId, requestedEpoch, requestedToken);
} finally {
lock.unlock();
}
}
private boolean resumeAllParticipants() {
boolean resumed = true;
for (int index = participants.size() - 1; index >= 0; index--) {
try {
participants.get(index).resume();
} catch (RuntimeException exception) {
resumed = false;
}
}
return resumed;
}
private MetadataMaintenanceLease completeAcquisition(Acquisition acquisition) {
lock.lock();
try {
if (!ownsAcquisition(acquisition)) {
throw MetadataMaintenanceException.operationConflict();
}
phase = MetadataMaintenancePhase.QUIESCED;
return lease(acquisition);
} finally {
lock.unlock();
}
}
private void rollback(Acquisition acquisition, List<MetadataMaintenanceParticipant> started) {
Collections.reverse(started);
boolean failed = false;
for (MetadataMaintenanceParticipant participant : started) {
try {
participant.resume();
} catch (RuntimeException exception) {
failed = true;
}
}
lock.lock();
try {
if (ownsAcquisition(acquisition) && !failed) {
reopen();
} else if (ownsAcquisition(acquisition)) {
phase = MetadataMaintenancePhase.RECOVERY_REQUIRED;
}
} finally {
lock.unlock();
}
}
private boolean ownsAcquisition(Acquisition acquisition) {
return phase == MetadataMaintenancePhase.QUIESCING
&& epoch == acquisition.epoch()
&& operationId.equals(acquisition.operationId())
&& leaseToken == acquisition.token();
}
private boolean ownsResumeLease(String resumedOperationId, long resumedEpoch, Object resumedToken) {
return (phase == MetadataMaintenancePhase.QUIESCED
|| phase == MetadataMaintenancePhase.QUIESCING)
&& epoch == resumedEpoch
&& operationId.equals(resumedOperationId)
&& leaseToken == resumedToken;
}
private MetadataMaintenanceLease lease(Acquisition acquisition) {
return new MetadataMaintenanceLease(
this, acquisition.operationId(), acquisition.epoch(), acquisition.token());
}
private void reopen() {
phase = MetadataMaintenancePhase.RUNNING;
operationId = null;
leaseToken = null;
}
private void requireOperationId(String requestedOperationId) {
if (requestedOperationId == null || requestedOperationId.isBlank()) {
throw MetadataMaintenanceException.invalidRequest();
}
}
private void validateParticipants(List<MetadataMaintenanceParticipant> registeredParticipants) {
Set<String> participantIds = new HashSet<>(registeredParticipants.size());
for (MetadataMaintenanceParticipant participant : registeredParticipants) {
String participantId = participant.participantId();
if (participantId == null || participantId.isBlank() || !participantIds.add(participantId)) {
throw MetadataMaintenanceException.invalidRequest();
}
}
}
private record Acquisition(String operationId, long epoch, Object token) {
}
}
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Stable, secret-free control-plane maintenance failure classifications. */
public enum MetadataMaintenanceErrorCode {
INVALID_REQUEST("invalid_request"),
OPERATION_CONFLICT("operation_conflict"),
QUIESCE_TIMEOUT("quiesce_timeout"),
QUIESCE_INTERRUPTED("quiesce_interrupted"),
PARTICIPANT_FAILURE("participant_failure"),
RESUME_FAILURE("resume_failure"),
STALE_LEASE("stale_lease");
private final String wireCode;
MetadataMaintenanceErrorCode(String wireCode) {
this.wireCode = wireCode;
}
public String wireCode() {
return wireCode;
}
}
@@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Safe maintenance failure that never exposes participant work or persistence details. */
public final class MetadataMaintenanceException extends RuntimeException {
private static final String INVALID_MESSAGE = "Metadata maintenance request is invalid";
private static final String CONFLICT_MESSAGE = "Metadata maintenance operation is already active";
private static final String TIMEOUT_MESSAGE = "Metadata producer drain timed out";
private static final String INTERRUPTED_MESSAGE = "Metadata producer drain was interrupted";
private static final String PARTICIPANT_MESSAGE = "Metadata producer could not be paused";
private static final String RESUME_MESSAGE = "Metadata producer could not be resumed";
private static final String STALE_MESSAGE = "Metadata maintenance lease is stale";
private final MetadataMaintenanceErrorCode code;
private MetadataMaintenanceException(MetadataMaintenanceErrorCode code, String message) {
super(message);
this.code = code;
}
public MetadataMaintenanceErrorCode code() {
return code;
}
public String safeMessage() {
return getMessage();
}
static MetadataMaintenanceException invalidRequest() {
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.INVALID_REQUEST, INVALID_MESSAGE);
}
static MetadataMaintenanceException operationConflict() {
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.OPERATION_CONFLICT, CONFLICT_MESSAGE);
}
public static MetadataMaintenanceException quiesceTimeout() {
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT, TIMEOUT_MESSAGE);
}
public static MetadataMaintenanceException quiesceInterrupted() {
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.QUIESCE_INTERRUPTED, INTERRUPTED_MESSAGE);
}
static MetadataMaintenanceException participantFailure() {
return new MetadataMaintenanceException(
MetadataMaintenanceErrorCode.PARTICIPANT_FAILURE, PARTICIPANT_MESSAGE);
}
static MetadataMaintenanceException resumeFailure() {
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.RESUME_FAILURE, RESUME_MESSAGE);
}
static MetadataMaintenanceException staleLease() {
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.STALE_LEASE, STALE_MESSAGE);
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Epoch-bound capability that resumes metadata producers exactly once. */
public final class MetadataMaintenanceLease implements AutoCloseable {
private final MetadataMaintenanceCoordinator coordinator;
private final String operationId;
private final long epoch;
private final Object token;
private boolean resumed;
MetadataMaintenanceLease(
MetadataMaintenanceCoordinator coordinator, String operationId, long epoch, Object token) {
this.coordinator = coordinator;
this.operationId = operationId;
this.epoch = epoch;
this.token = token;
}
public synchronized void resume() {
if (resumed) {
return;
}
coordinator.resume(operationId, epoch, token);
resumed = true;
}
@Override
public void close() {
resume();
}
}
@@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
/** One metadata producer that can stop admitting work and drain already admitted work. */
public interface MetadataMaintenanceParticipant {
String participantId();
/** Stop admitting work and drain work admitted before this call. */
void quiesce(Duration timeout);
/** Resume normal admission without creating another scheduler or consumer. */
void resume();
}
@@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Lifecycle of process-local metadata producer admission. */
public enum MetadataMaintenancePhase {
RUNNING,
QUIESCING,
QUIESCED,
RECOVERY_REQUIRED
}
@@ -0,0 +1,13 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Read-only process-local maintenance state without the lease capability. */
public record MetadataMaintenanceSnapshot(
MetadataMaintenancePhase phase, String operationId, long epoch) {
}
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate;
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/** Fail-closed fallbacks for maintenance facts that the runtime cannot prove. */
@Configuration(proxyBeanMethods = false)
public class MigrationGuardConfiguration {
@Bean
@ConditionalOnNormalBusinessRuntime
@ConditionalOnBean(StandaloneDeploymentOwnerView.class)
@ConditionalOnMissingBean(InstallationConvergenceVerifier.class)
InstallationConvergenceVerifier normalInstallationConvergenceVerifier(
InstallationRecordRepository records, StandaloneDeploymentOwnerView owner) {
return new NormalInstallationConvergenceVerifier(records, owner);
}
@Bean
@ConditionalOnBean({StandaloneDeploymentOwnerView.class, InstallationConvergenceVerifier.class})
@ConditionalOnMissingBean(DeploymentSingletonAuthority.class)
DeploymentSingletonAuthority deploymentSingletonAuthority(
BusinessRuntimeGate runtimeGate,
StandaloneDeploymentOwnerView owner,
InstallationConvergenceVerifier convergence) {
if (runtimeGate.isOpen()) {
return new StandaloneDeploymentSingletonAuthority(owner, convergence);
}
return unavailableDeploymentSingletonAuthority();
}
@Bean
@ConditionalOnMissingBean(DeploymentSingletonAuthority.class)
DeploymentSingletonAuthority unavailableDeploymentSingletonAuthority() {
return (operationId, timeout) -> {
throw MigrationMaintenanceException.deploymentAuthorityUnavailable();
};
}
@Bean
@ConditionalOnMissingBean(MigrationSourceGuard.class)
MigrationSourceGuard unavailableMigrationSourceGuard() {
return (operationId, timeout) -> {
throw MigrationMaintenanceException.sourceUnavailable();
};
}
}
@@ -0,0 +1,21 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Stable safe failure categories for migration maintenance acquisition and release. */
public enum MigrationMaintenanceErrorCode {
MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE,
MIGRATION_SOURCE_UNAVAILABLE,
MIGRATION_MULTI_NODE_UNSUPPORTED,
MIGRATION_OPERATION_CONFLICT,
MIGRATION_MAINTENANCE_TIMEOUT,
MIGRATION_MAINTENANCE_INTERRUPTED,
MIGRATION_MAINTENANCE_FAILURE,
MIGRATION_RESUME_FAILURE,
INVALID_REQUEST
}
@@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Secret-free migration maintenance failure. */
public final class MigrationMaintenanceException extends RuntimeException {
private final MigrationMaintenanceErrorCode code;
private MigrationMaintenanceException(MigrationMaintenanceErrorCode code, String message) {
super(message);
this.code = code;
}
public MigrationMaintenanceErrorCode code() {
return code;
}
public String safeMessage() {
return getMessage();
}
public static MigrationMaintenanceException deploymentAuthorityUnavailable() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE,
"Migration deployment authority is unavailable");
}
public static MigrationMaintenanceException sourceUnavailable() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE,
"Migration metadata source is unavailable");
}
public static MigrationMaintenanceException multiNodeUnsupported() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED,
"Multi-node metadata migration is unsupported");
}
public static MigrationMaintenanceException operationConflict() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT,
"Migration maintenance operation is already active");
}
static MigrationMaintenanceException timeout() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_TIMEOUT,
"Migration maintenance acquisition timed out");
}
static MigrationMaintenanceException interrupted() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_INTERRUPTED,
"Migration maintenance acquisition was interrupted");
}
public static MigrationMaintenanceException maintenanceFailure() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_FAILURE,
"Migration maintenance acquisition failed");
}
static MigrationMaintenanceException resumeFailure() {
return failure(MigrationMaintenanceErrorCode.MIGRATION_RESUME_FAILURE,
"Migration maintenance release failed");
}
public static MigrationMaintenanceException invalidRequest() {
return failure(MigrationMaintenanceErrorCode.INVALID_REQUEST,
"Migration maintenance request is invalid");
}
private static MigrationMaintenanceException failure(
MigrationMaintenanceErrorCode code, String message) {
return new MigrationMaintenanceException(code, message);
}
}
@@ -0,0 +1,18 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Owner capability for one fully acquired migration maintenance window. */
public interface MigrationMaintenanceLease extends AutoCloseable {
/** Runs synchronous work against the exact source fenced by this maintenance window. */
void withSourceConnection(MigrationSourceAction action);
@Override
void close();
}
@@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
/** Acquires the complete process-local maintenance window required before metadata migration. */
public interface MigrationMaintenanceOrchestrator {
MigrationMaintenanceLease acquire(String operationId, Duration timeout);
}
@@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.sql.Connection;
/** Synchronous work scoped to the exact metadata source held by a maintenance lease. */
@FunctionalInterface
public interface MigrationSourceAction {
/**
* Uses the guarded source only for this callback. The action must not retain, replace, or
* independently close the connection. Only the bounded JDBC migration executor may invalidate
* it on a fail-closed timeout or unknown-outcome path; final ownership remains with the lease.
*/
void execute(Connection source);
}
@@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.time.Duration;
/** Fences a metadata source whose local access mode is safe for migration. */
public interface MigrationSourceGuard {
MigrationSourceLease fence(String operationId, Duration timeout);
}
@@ -0,0 +1,18 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
/** Capability that releases one safe metadata-source fence. */
public interface MigrationSourceLease extends AutoCloseable {
/** Runs synchronous work against the exact source owned by this lease. */
void withConnection(MigrationSourceAction action);
@Override
void close();
}
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.nio.file.Path;
import org.apache.hertzbeat.manager.setup.installation.InstallationConvergenceService;
import org.apache.hertzbeat.manager.setup.installation.InstallationMode;
import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository;
/** Normal-context adapter that re-reads fingerprint and database installation state. */
public final class NormalInstallationConvergenceVerifier implements InstallationConvergenceVerifier {
private final InstallationRecordRepository records;
private final StandaloneDeploymentOwnerView owner;
public NormalInstallationConvergenceVerifier(
InstallationRecordRepository records, StandaloneDeploymentOwnerView owner) {
this.records = records;
this.owner = owner;
}
@Override
public boolean isFullyConverged() {
Path root = owner.installationRoot();
return new InstallationConvergenceService(
records, root, root.resolve("data/config/.installation-fingerprint")).classify()
== InstallationMode.FULL;
}
}
@@ -0,0 +1,18 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0.
*/
package org.apache.hertzbeat.manager.maintenance;
import java.nio.file.Path;
/** Non-owning view of the process-level standalone deployment lock. */
public interface StandaloneDeploymentOwnerView {
Path installationRoot();
boolean isValid();
}

Some files were not shown because too many files have changed in this diff Show More