mirror of
https://github.com/dromara/hertzbeat.git
synced 2026-09-17 09:40:58 +00:00
Merge managed setup runtime into 2.0.0
This commit is contained in:
@@ -19,7 +19,6 @@
|
||||
package org.apache.hertzbeat.ai.config;
|
||||
|
||||
import com.openai.client.OpenAIClient;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -58,8 +57,7 @@ public class LlmConfig {
|
||||
this.applicationContext = applicationContext;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void registerInitialChatClient() {
|
||||
void registerInitialChatClient() {
|
||||
registerChatClient();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.ai.config;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Registers the configured chat client only after the business runtime opens. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class LlmConfigInitializer implements CommandLineRunner {
|
||||
|
||||
private final LlmConfig llmConfig;
|
||||
|
||||
public LlmConfigInitializer(LlmConfig llmConfig) {
|
||||
this.llmConfig = llmConfig;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
llmConfig.registerInitialChatClient();
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,7 @@ import org.apache.hertzbeat.ai.sop.registry.SkillRegistry;
|
||||
import org.apache.hertzbeat.ai.utils.SopMessageUtil;
|
||||
import org.apache.hertzbeat.common.entity.ai.ChatMessage;
|
||||
import org.apache.hertzbeat.common.entity.ai.SopSchedule;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
@@ -43,6 +44,7 @@ import org.springframework.stereotype.Component;
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@EnableScheduling
|
||||
public class SopScheduleExecutor {
|
||||
|
||||
|
||||
+193
-83
@@ -20,8 +20,8 @@ package org.apache.hertzbeat.alert.calculate.periodic;
|
||||
import static org.apache.hertzbeat.common.constants.CommonConstants.LOG_ALERT_THRESHOLD_TYPE_PERIODIC;
|
||||
import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC;
|
||||
import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC;
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@@ -37,28 +37,30 @@ import java.util.concurrent.TimeUnit;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Periodic Alert Rule Scheduler
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class PeriodicAlertRuleScheduler implements CommandLineRunner, DisposableBean {
|
||||
public class PeriodicAlertRuleScheduler {
|
||||
|
||||
private final MetricsPeriodicAlertCalculator metricsCalculator;
|
||||
private final LogPeriodicAlertCalculator logCalculator;
|
||||
private final TracePeriodicAlertCalculator traceCalculator;
|
||||
private final AlertDefineDao alertDefineDao;
|
||||
private final ScheduledExecutorService scheduledExecutor;
|
||||
private final ExecutorService periodicExecutor;
|
||||
private final Semaphore periodicPermits;
|
||||
private final boolean virtualThreadsEnabled;
|
||||
private ScheduledExecutorService scheduledExecutor;
|
||||
private ExecutorService periodicExecutor;
|
||||
private Semaphore periodicPermits;
|
||||
private final VirtualThreadProperties virtualThreadProperties;
|
||||
private boolean virtualThreadsEnabled;
|
||||
private final Map<Long, ScheduledTaskState> scheduledTasks;
|
||||
private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate();
|
||||
private boolean maintenancePaused;
|
||||
|
||||
@Autowired
|
||||
public PeriodicAlertRuleScheduler(MetricsPeriodicAlertCalculator metricsCalculator,
|
||||
@@ -70,10 +72,15 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
this.logCalculator = logCalculator;
|
||||
this.traceCalculator = traceCalculator;
|
||||
this.alertDefineDao = alertDefineDao;
|
||||
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
|
||||
log.error("Scheduled periodic alert threshold has uncaughtException.");
|
||||
log.error(throwable.getMessage(), throwable);
|
||||
};
|
||||
this.virtualThreadProperties = virtualThreadProperties == null
|
||||
? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
this.scheduledTasks = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
synchronized void start() {
|
||||
if (scheduledExecutor != null) {
|
||||
return;
|
||||
}
|
||||
ThreadFactory threadFactory = new ThreadFactoryBuilder()
|
||||
.setUncaughtExceptionHandler((thread, throwable) -> {
|
||||
log.error("Scheduled periodic alert threshold has uncaughtException.");
|
||||
@@ -82,49 +89,18 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
.setDaemon(true)
|
||||
.setNameFormat("periodic-alert-threshold-worker-%d")
|
||||
.build();
|
||||
this.scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory);
|
||||
VirtualThreadProperties properties =
|
||||
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
this.virtualThreadsEnabled = properties.enabled();
|
||||
int maxConcurrentPeriodicTasks = Math.max(1, properties.alerter().periodicMaxConcurrentJobs());
|
||||
scheduledExecutor = Executors.newScheduledThreadPool(10, threadFactory);
|
||||
virtualThreadsEnabled = virtualThreadProperties.enabled();
|
||||
int maxConcurrentPeriodicTasks = Math.max(
|
||||
1, virtualThreadProperties.alerter().periodicMaxConcurrentJobs());
|
||||
this.periodicExecutor = virtualThreadsEnabled
|
||||
? Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
.name("periodic-alert-task-", 0)
|
||||
.uncaughtExceptionHandler(handler)
|
||||
.uncaughtExceptionHandler((thread, throwable) ->
|
||||
log.error("Periodic alert task failed", throwable))
|
||||
.factory())
|
||||
: null;
|
||||
this.periodicPermits = virtualThreadsEnabled ? new Semaphore(maxConcurrentPeriodicTasks) : null;
|
||||
this.scheduledTasks = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
public void cancelSchedule(Long ruleId) {
|
||||
if (ruleId == null) {
|
||||
return;
|
||||
}
|
||||
ScheduledTaskState state = scheduledTasks.remove(ruleId);
|
||||
if (state != null) {
|
||||
state.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
public void updateSchedule(AlertDefine rule) {
|
||||
if (rule == null || rule.getId() == null) {
|
||||
log.error("Alert rule is null or rule id is null.");
|
||||
return;
|
||||
}
|
||||
cancelSchedule(rule.getId());
|
||||
if (isPeriodicRule(rule.getType())) {
|
||||
ScheduledTaskState state = new ScheduledTaskState(rule);
|
||||
ScheduledFuture<?> future = scheduledExecutor.scheduleAtFixedRate(
|
||||
virtualThreadsEnabled ? state::trigger : () -> executeRule(rule),
|
||||
0, rule.getPeriod(), TimeUnit.SECONDS);
|
||||
state.setScheduledFuture(future);
|
||||
scheduledTasks.put(rule.getId(), state);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) throws Exception {
|
||||
log.info("Starting periodic alert rule scheduler...");
|
||||
List<AlertDefine> metricsPeriodicRules = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC);
|
||||
List<AlertDefine> logPeriodicRules = alertDefineDao.findAlertDefinesByTypeAndEnableTrue(LOG_ALERT_THRESHOLD_TYPE_PERIODIC);
|
||||
@@ -139,13 +115,50 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
synchronized void stop() {
|
||||
scheduledTasks.values().forEach(ScheduledTaskState::cancel);
|
||||
scheduledTasks.clear();
|
||||
scheduledExecutor.shutdownNow();
|
||||
if (scheduledExecutor != null) {
|
||||
scheduledExecutor.shutdownNow();
|
||||
scheduledExecutor = null;
|
||||
}
|
||||
if (periodicExecutor != null) {
|
||||
periodicExecutor.shutdownNow();
|
||||
periodicExecutor = null;
|
||||
}
|
||||
periodicPermits = null;
|
||||
}
|
||||
|
||||
public synchronized void cancelSchedule(Long ruleId) {
|
||||
if (ruleId == null || scheduledExecutor == null) {
|
||||
return;
|
||||
}
|
||||
ScheduledTaskState state = scheduledTasks.remove(ruleId);
|
||||
if (state != null) {
|
||||
state.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void updateSchedule(AlertDefine rule) {
|
||||
if (rule == null || rule.getId() == null) {
|
||||
log.error("Alert rule is null or rule id is null.");
|
||||
return;
|
||||
}
|
||||
if (scheduledExecutor == null) {
|
||||
return;
|
||||
}
|
||||
cancelSchedule(rule.getId());
|
||||
if (isPeriodicRule(rule.getType())) {
|
||||
ScheduledExecutorService currentScheduledExecutor = scheduledExecutor;
|
||||
ExecutorService currentPeriodicExecutor = periodicExecutor;
|
||||
Semaphore currentPeriodicPermits = periodicPermits;
|
||||
ScheduledTaskState state = new ScheduledTaskState(
|
||||
rule, currentPeriodicExecutor, currentPeriodicPermits);
|
||||
ScheduledFuture<?> future = currentScheduledExecutor.scheduleAtFixedRate(
|
||||
state::trigger,
|
||||
0, rule.getPeriod(), TimeUnit.SECONDS);
|
||||
state.setScheduledFuture(future);
|
||||
scheduledTasks.put(rule.getId(), state);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,6 +172,48 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void pauseAdmission() {
|
||||
maintenancePaused = true;
|
||||
maintenanceGate.pauseAdmission();
|
||||
}
|
||||
|
||||
public void awaitDrained(long timeoutNanos) throws InterruptedException, java.util.concurrent.TimeoutException {
|
||||
maintenanceGate.awaitDrained(timeoutNanos);
|
||||
}
|
||||
|
||||
public void resumeAdmission() {
|
||||
List<ScheduledTaskState> states;
|
||||
synchronized (this) {
|
||||
if (!maintenancePaused) {
|
||||
return;
|
||||
}
|
||||
maintenanceGate.resumeAdmission();
|
||||
maintenancePaused = false;
|
||||
states = new ArrayList<>(scheduledTasks.values());
|
||||
}
|
||||
states.forEach(ScheduledTaskState::resumeMissed);
|
||||
}
|
||||
|
||||
private synchronized WorkAdmissionGate.Permit acquireTriggerPermit(ScheduledTaskState state) {
|
||||
if (maintenancePaused) {
|
||||
state.markMissed();
|
||||
return null;
|
||||
}
|
||||
return maintenanceGate.tryAcquire();
|
||||
}
|
||||
|
||||
void beforeRuleTrigger(AlertDefine rule) {
|
||||
}
|
||||
|
||||
private void executeRuleWithPermit(AlertDefine rule, WorkAdmissionGate.Permit permit) {
|
||||
if (permit == null) {
|
||||
return;
|
||||
}
|
||||
try (permit) {
|
||||
executeRule(rule);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isPeriodicRule(String type) {
|
||||
return METRIC_ALERT_THRESHOLD_TYPE_PERIODIC.equals(type)
|
||||
|| LOG_ALERT_THRESHOLD_TYPE_PERIODIC.equals(type)
|
||||
@@ -168,35 +223,59 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
private final class ScheduledTaskState {
|
||||
|
||||
private final AlertDefine rule;
|
||||
private final ExecutorService taskExecutor;
|
||||
private final Semaphore taskPermits;
|
||||
private ScheduledFuture<?> scheduledFuture;
|
||||
private Future<?> runningFuture;
|
||||
private boolean running;
|
||||
private boolean pending;
|
||||
private WorkAdmissionGate.Permit pendingPermit;
|
||||
private boolean cancelled;
|
||||
private boolean missedWhilePaused;
|
||||
|
||||
private ScheduledTaskState(AlertDefine rule) {
|
||||
private ScheduledTaskState(AlertDefine rule, ExecutorService taskExecutor, Semaphore taskPermits) {
|
||||
this.rule = rule;
|
||||
this.taskExecutor = taskExecutor;
|
||||
this.taskPermits = taskPermits;
|
||||
}
|
||||
|
||||
private synchronized void setScheduledFuture(ScheduledFuture<?> scheduledFuture) {
|
||||
this.scheduledFuture = scheduledFuture;
|
||||
}
|
||||
|
||||
private synchronized void trigger() {
|
||||
if (cancelled) {
|
||||
private void trigger() {
|
||||
beforeRuleTrigger(rule);
|
||||
WorkAdmissionGate.Permit permit = acquireTriggerPermit(this);
|
||||
if (permit == null) {
|
||||
return;
|
||||
}
|
||||
if (running) {
|
||||
pending = true;
|
||||
return;
|
||||
synchronized (this) {
|
||||
if (cancelled) {
|
||||
permit.close();
|
||||
return;
|
||||
}
|
||||
if (running) {
|
||||
if (!pending) {
|
||||
pending = true;
|
||||
pendingPermit = permit;
|
||||
} else {
|
||||
permit.close();
|
||||
}
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
}
|
||||
running = true;
|
||||
submitLocked();
|
||||
submit(permit);
|
||||
}
|
||||
|
||||
private synchronized void cancel() {
|
||||
cancelled = true;
|
||||
pending = false;
|
||||
if (pendingPermit != null) {
|
||||
pendingPermit.close();
|
||||
pendingPermit = null;
|
||||
}
|
||||
missedWhilePaused = false;
|
||||
ScheduledFuture<?> periodicFuture = scheduledFuture;
|
||||
Future<?> currentFuture = runningFuture;
|
||||
if (periodicFuture != null) {
|
||||
@@ -207,33 +286,46 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
}
|
||||
}
|
||||
|
||||
private void submitLocked() {
|
||||
private void submit(WorkAdmissionGate.Permit permit) {
|
||||
if (taskExecutor == null) {
|
||||
runTask(permit);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
runningFuture = periodicExecutor.submit(() -> {
|
||||
boolean permitAcquired = false;
|
||||
try {
|
||||
periodicPermits.acquire();
|
||||
permitAcquired = true;
|
||||
if (!Thread.currentThread().isInterrupted()) {
|
||||
executeRule(rule);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (Exception e) {
|
||||
log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e);
|
||||
} finally {
|
||||
if (permitAcquired) {
|
||||
periodicPermits.release();
|
||||
}
|
||||
onComplete();
|
||||
}
|
||||
});
|
||||
runningFuture = taskExecutor.submit(() -> runTask(permit));
|
||||
} catch (RuntimeException e) {
|
||||
running = false;
|
||||
permit.close();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void runTask(WorkAdmissionGate.Permit permit) {
|
||||
boolean concurrencyPermitAcquired = false;
|
||||
try {
|
||||
if (taskPermits != null) {
|
||||
taskPermits.acquire();
|
||||
concurrencyPermitAcquired = true;
|
||||
}
|
||||
if (!Thread.currentThread().isInterrupted()) {
|
||||
executeRuleWithPermit(rule, permit);
|
||||
permit = null;
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (Exception e) {
|
||||
log.error("Periodic alert rule {} execution error: {}", rule.getName(), e.getMessage(), e);
|
||||
} finally {
|
||||
if (permit != null) {
|
||||
permit.close();
|
||||
}
|
||||
if (concurrencyPermitAcquired) {
|
||||
taskPermits.release();
|
||||
}
|
||||
onComplete();
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void onComplete() {
|
||||
runningFuture = null;
|
||||
if (cancelled) {
|
||||
@@ -246,7 +338,25 @@ public class PeriodicAlertRuleScheduler implements CommandLineRunner, Disposable
|
||||
return;
|
||||
}
|
||||
pending = false;
|
||||
submitLocked();
|
||||
WorkAdmissionGate.Permit nextPermit = pendingPermit;
|
||||
pendingPermit = null;
|
||||
submit(nextPermit);
|
||||
}
|
||||
|
||||
private synchronized void markMissed() {
|
||||
if (!cancelled) {
|
||||
missedWhilePaused = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void resumeMissed() {
|
||||
synchronized (this) {
|
||||
if (!missedWhilePaused || cancelled) {
|
||||
return;
|
||||
}
|
||||
missedWhilePaused = false;
|
||||
}
|
||||
trigger();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.alert.calculate.periodic;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Owns scheduler threads and database startup reads only in normal business runtime. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class PeriodicAlertRuleSchedulerLifecycle implements CommandLineRunner, DisposableBean {
|
||||
|
||||
private final PeriodicAlertRuleScheduler scheduler;
|
||||
|
||||
public PeriodicAlertRuleSchedulerLifecycle(PeriodicAlertRuleScheduler scheduler) {
|
||||
this.scheduler = scheduler;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
scheduler.start();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
scheduler.stop();
|
||||
}
|
||||
}
|
||||
+3
-1
@@ -137,8 +137,10 @@ public class MetricsRealTimeAlertCalculator {
|
||||
continue;
|
||||
}
|
||||
backoff.reset();
|
||||
calculate(metricsData);
|
||||
// The telemetry handoff precedes alert reduction so maintenance backpressure cannot
|
||||
// occupy every calculator before later samples reach storage.
|
||||
dataQueue.sendMetricsDataToStorage(metricsData);
|
||||
calculate(metricsData);
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (CommonDataQueueUnknownException ue) {
|
||||
|
||||
+2
@@ -25,6 +25,7 @@ import org.apache.hertzbeat.alert.calculate.realtime.window.LogWorker;
|
||||
import org.apache.hertzbeat.alert.calculate.realtime.window.TimeService;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
|
||||
import org.apache.hertzbeat.common.util.BackoffUtils;
|
||||
import org.apache.hertzbeat.common.util.ExponentialBackoff;
|
||||
@@ -44,6 +45,7 @@ import java.util.concurrent.TimeUnit;
|
||||
* 4. Distributing logs to workers
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Slf4j
|
||||
public class WindowedLogRealTimeAlertCalculator implements Runnable {
|
||||
|
||||
|
||||
+3
-1
@@ -23,6 +23,7 @@ import org.apache.hertzbeat.alert.calculate.JexlExprCalculator;
|
||||
import org.apache.hertzbeat.alert.service.AlertDefineService;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -38,6 +39,7 @@ import java.util.Map;
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public class LogWorker {
|
||||
|
||||
private static final String LOG_PREFIX = "log";
|
||||
@@ -120,4 +122,4 @@ public class LogWorker {
|
||||
}
|
||||
return System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -24,6 +24,7 @@ import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
@@ -42,6 +43,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
||||
* 3. Broadcasting watermarks to all subscribers (WindowAggregator)
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Slf4j
|
||||
public class TimeService {
|
||||
|
||||
|
||||
+2
@@ -24,6 +24,7 @@ import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
@@ -49,6 +50,7 @@ import java.util.concurrent.TimeUnit;
|
||||
* 4. Sending closed windows to AlarmEvaluator
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Slf4j
|
||||
public class WindowAggregator implements TimeService.WatermarkListener, Runnable {
|
||||
|
||||
|
||||
+23
-12
@@ -106,18 +106,29 @@ public class AlertNoticeDispatch {
|
||||
return Optional.ofNullable(noticeConfigService.getReceiverFilterRule(alert));
|
||||
}
|
||||
|
||||
public void dispatchAlarm(GroupAlert groupAlert) {
|
||||
if (groupAlert != null) {
|
||||
// Determining alarm type storage
|
||||
GroupAlert storedGroupAlert = alertStoreHandler.store(groupAlert);
|
||||
// Notice distribution
|
||||
sendNotify(storedGroupAlert);
|
||||
// Execute the plugin if enable (Compatible with old version plugins, will be removed in later versions)
|
||||
pluginRunner.pluginExecute(Plugin.class, plugin -> plugin.alert(storedGroupAlert));
|
||||
// Execute the plugin if enable with params
|
||||
pluginRunner.pluginExecute(PostAlertPlugin.class, (afterAlertPlugin, pluginContext) -> afterAlertPlugin.execute(storedGroupAlert, pluginContext));
|
||||
// Send alert to the sse client
|
||||
emitterManager.broadcast(JsonUtil.toJson(storedGroupAlert));
|
||||
public boolean dispatchAlarm(GroupAlert groupAlert) {
|
||||
if (groupAlert == null) {
|
||||
return false;
|
||||
}
|
||||
GroupAlert storedGroupAlert = alertStoreHandler.store(groupAlert);
|
||||
dispatchAfterStore(storedGroupAlert);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void dispatchAfterStore(GroupAlert storedGroupAlert) {
|
||||
runAfterStore(() -> sendNotify(storedGroupAlert), "notice");
|
||||
runAfterStore(() -> pluginRunner.pluginExecute(
|
||||
Plugin.class, plugin -> plugin.alert(storedGroupAlert)), "legacy-plugin");
|
||||
runAfterStore(() -> pluginRunner.pluginExecute(PostAlertPlugin.class,
|
||||
(plugin, context) -> plugin.execute(storedGroupAlert, context)), "post-plugin");
|
||||
runAfterStore(() -> emitterManager.broadcast(JsonUtil.toJson(storedGroupAlert)), "sse");
|
||||
}
|
||||
|
||||
private void runAfterStore(Runnable action, String stage) {
|
||||
try {
|
||||
action.run();
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Post-store alert dispatch failed at stage: {}", stage);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+4
-1
@@ -54,6 +54,9 @@ public class EmailAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
|
||||
@Value("${spring.mail.username:demo}")
|
||||
private String username;
|
||||
|
||||
@Value("${hertzbeat.mail.from-address:${spring.mail.username:demo}}")
|
||||
private String fromAddress;
|
||||
|
||||
@Value("${spring.mail.password:demo}")
|
||||
private String password;
|
||||
|
||||
@@ -82,7 +85,7 @@ public class EmailAlertNotifyHandlerImpl extends AbstractAlertNotifyHandlerImpl
|
||||
try {
|
||||
// get sender
|
||||
JavaMailSenderImpl sender = (JavaMailSenderImpl) javaMailSender;
|
||||
String fromUsername = username;
|
||||
String fromUsername = fromAddress;
|
||||
try {
|
||||
boolean useDatabase = false;
|
||||
GeneralConfig emailConfig = generalConfigDao.findByType(TYPE);
|
||||
|
||||
+110
-2
@@ -18,12 +18,17 @@
|
||||
package org.apache.hertzbeat.alert.reduce;
|
||||
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
|
||||
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
|
||||
import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
@@ -44,6 +49,14 @@ public class AlarmCommonReduce implements DisposableBean {
|
||||
|
||||
private final ApplicationEventPublisher eventPublisher;
|
||||
|
||||
private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate();
|
||||
|
||||
private final ReentrantLock maintenanceLock = new ReentrantLock(true);
|
||||
|
||||
private final Deque<Runnable> deferredTasks = new ArrayDeque<>();
|
||||
|
||||
private boolean stopped;
|
||||
|
||||
public AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce) {
|
||||
this(alarmGroupReduce, VirtualThreadProperties.defaults(), event -> { });
|
||||
}
|
||||
@@ -62,6 +75,12 @@ public class AlarmCommonReduce implements DisposableBean {
|
||||
this.workerExecutor = initWorkExecutor(properties);
|
||||
}
|
||||
|
||||
AlarmCommonReduce(AlarmGroupReduce alarmGroupReduce, ManagedExecutor workerExecutor) {
|
||||
this.alarmGroupReduce = alarmGroupReduce;
|
||||
this.workerExecutor = workerExecutor;
|
||||
this.eventPublisher = event -> { };
|
||||
}
|
||||
|
||||
private ManagedExecutor initWorkExecutor(VirtualThreadProperties properties) {
|
||||
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
|
||||
log.error("alerter-reduce-worker has uncaughtException.");
|
||||
@@ -87,11 +106,11 @@ public class AlarmCommonReduce implements DisposableBean {
|
||||
|
||||
|
||||
public void reduceAndSendAlarm(SingleAlert alert) {
|
||||
workerExecutor.execute(reduceAlarmTask(alert));
|
||||
submitOrDefer(reduceAlarmTask(alert));
|
||||
}
|
||||
|
||||
public void reduceAndSendAlarmGroup(Map<String, String> groupLabels, List<SingleAlert> alerts) {
|
||||
workerExecutor.execute(() -> {
|
||||
submitOrDefer(() -> {
|
||||
try {
|
||||
// Generate alert fingerprint
|
||||
for (SingleAlert alert : alerts) {
|
||||
@@ -107,6 +126,87 @@ public class AlarmCommonReduce implements DisposableBean {
|
||||
});
|
||||
}
|
||||
|
||||
public void pauseAdmission() {
|
||||
maintenanceLock.lock();
|
||||
try {
|
||||
maintenanceGate.pauseAdmission();
|
||||
} finally {
|
||||
maintenanceLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException {
|
||||
maintenanceGate.awaitDrained(timeoutNanos);
|
||||
}
|
||||
|
||||
public void resumeAdmission() {
|
||||
maintenanceLock.lock();
|
||||
try {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
while (!deferredTasks.isEmpty()) {
|
||||
Runnable deferred = deferredTasks.peekFirst();
|
||||
WorkAdmissionGate.Permit permit = maintenanceGate.reserveReplay();
|
||||
if (permit == null) {
|
||||
return;
|
||||
}
|
||||
submitAdmitted(deferred, permit);
|
||||
deferredTasks.removeFirst();
|
||||
}
|
||||
maintenanceGate.resumeAdmission();
|
||||
} finally {
|
||||
maintenanceLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void submitOrDefer(Runnable task) {
|
||||
maintenanceLock.lock();
|
||||
try {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
WorkAdmissionGate.Permit permit = maintenanceGate.tryAcquire();
|
||||
if (permit != null) {
|
||||
beforeAdmittedSubmission();
|
||||
submitAdmitted(task, permit);
|
||||
return;
|
||||
}
|
||||
deferredTasks.addLast(task);
|
||||
} finally {
|
||||
maintenanceLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void beforeAdmittedSubmission() {
|
||||
}
|
||||
|
||||
boolean hasQueuedMaintenanceThread(Thread thread) {
|
||||
return maintenanceLock.hasQueuedThread(thread);
|
||||
}
|
||||
|
||||
int deferredTaskCount() {
|
||||
maintenanceLock.lock();
|
||||
try {
|
||||
return deferredTasks.size();
|
||||
} finally {
|
||||
maintenanceLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void submitAdmitted(Runnable task, WorkAdmissionGate.Permit permit) {
|
||||
try {
|
||||
workerExecutor.execute(() -> {
|
||||
try (permit) {
|
||||
task.run();
|
||||
}
|
||||
});
|
||||
} catch (RuntimeException exception) {
|
||||
permit.close();
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
Runnable reduceAlarmTask(SingleAlert alert) {
|
||||
return () -> {
|
||||
try {
|
||||
@@ -138,6 +238,14 @@ public class AlarmCommonReduce implements DisposableBean {
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
maintenanceLock.lock();
|
||||
try {
|
||||
maintenanceGate.stop();
|
||||
deferredTasks.clear();
|
||||
stopped = true;
|
||||
} finally {
|
||||
maintenanceLock.unlock();
|
||||
}
|
||||
workerExecutor.close();
|
||||
}
|
||||
}
|
||||
|
||||
+174
-57
@@ -31,6 +31,7 @@ import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.Data;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@@ -93,57 +94,100 @@ public class AlarmGroupReduce implements DisposableBean {
|
||||
*/
|
||||
private final Map<String, GroupAlertCache> groupCacheMap;
|
||||
|
||||
private final ScheduledExecutorService scheduledExecutor;
|
||||
private final AlertGroupConvergeDao alertGroupConvergeDao;
|
||||
private final VirtualThreadProperties virtualThreadProperties;
|
||||
private ScheduledExecutorService scheduledExecutor;
|
||||
|
||||
private final ExecutorService workerExecutor;
|
||||
private ExecutorService workerExecutor;
|
||||
|
||||
private final ScheduledDispatchTask checkTask;
|
||||
private ScheduledDispatchTask checkTask;
|
||||
|
||||
public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao) {
|
||||
this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults(), true);
|
||||
this(alarmInhibitReduce, alertGroupConvergeDao, VirtualThreadProperties.defaults());
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
|
||||
VirtualThreadProperties virtualThreadProperties) {
|
||||
this(alarmInhibitReduce, alertGroupConvergeDao, virtualThreadProperties, true);
|
||||
}
|
||||
|
||||
AlarmGroupReduce(AlarmInhibitReduce alarmInhibitReduce, AlertGroupConvergeDao alertGroupConvergeDao,
|
||||
VirtualThreadProperties virtualThreadProperties, boolean autoStart) {
|
||||
this.alarmInhibitReduce = alarmInhibitReduce;
|
||||
this.groupDefines = new ConcurrentHashMap<>(8);
|
||||
this.groupCacheMap = new ConcurrentHashMap<>(8);
|
||||
VirtualThreadProperties properties =
|
||||
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
this.scheduledExecutor = createScheduler();
|
||||
this.workerExecutor = createVirtualExecutor(properties);
|
||||
this.checkTask = new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups);
|
||||
List<AlertGroupConverge> groupConverges = alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue();
|
||||
refreshGroupDefines(groupConverges);
|
||||
if (autoStart) {
|
||||
startCheckAndSendGroups();
|
||||
}
|
||||
this.alertGroupConvergeDao = alertGroupConvergeDao;
|
||||
this.virtualThreadProperties = virtualThreadProperties == null
|
||||
? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
}
|
||||
|
||||
private void startCheckAndSendGroups() {
|
||||
scheduledExecutor.scheduleAtFixedRate(this::dispatchCheckAndSendGroups, 10000, CHECK_INTERVAL,
|
||||
synchronized void start() {
|
||||
if (scheduledExecutor != null) {
|
||||
return;
|
||||
}
|
||||
scheduledExecutor = createScheduler();
|
||||
workerExecutor = createVirtualExecutor(virtualThreadProperties);
|
||||
ScheduledDispatchTask currentCheckTask =
|
||||
new ScheduledDispatchTask(workerExecutor, this::runCheckAndSendGroups);
|
||||
checkTask = currentCheckTask;
|
||||
refreshGroupDefines(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue());
|
||||
startCheckAndSendGroups(currentCheckTask);
|
||||
}
|
||||
|
||||
private void startCheckAndSendGroups(ScheduledDispatchTask currentCheckTask) {
|
||||
scheduledExecutor.scheduleAtFixedRate(currentCheckTask::dispatch, 10000, CHECK_INTERVAL,
|
||||
TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
void dispatchCheckAndSendGroups() {
|
||||
checkTask.dispatch();
|
||||
synchronized void dispatchCheckAndSendGroups() {
|
||||
if (checkTask != null) {
|
||||
checkTask.dispatch();
|
||||
}
|
||||
}
|
||||
|
||||
void beforeCheckAndSendGroupsRun() {
|
||||
}
|
||||
|
||||
public void pauseAdmission() {
|
||||
ScheduledDispatchTask currentTask;
|
||||
synchronized (this) {
|
||||
currentTask = checkTask;
|
||||
}
|
||||
if (currentTask != null) {
|
||||
currentTask.pauseAdmission();
|
||||
}
|
||||
}
|
||||
|
||||
public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException {
|
||||
ScheduledDispatchTask currentTask;
|
||||
synchronized (this) {
|
||||
currentTask = checkTask;
|
||||
}
|
||||
if (currentTask != null) {
|
||||
currentTask.awaitDrained(timeoutNanos);
|
||||
}
|
||||
}
|
||||
|
||||
public void resumeAdmission() {
|
||||
ScheduledDispatchTask currentTask;
|
||||
synchronized (this) {
|
||||
currentTask = checkTask;
|
||||
}
|
||||
if (currentTask != null) {
|
||||
currentTask.resumeAdmission();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
scheduledExecutor.shutdownNow();
|
||||
public synchronized void destroy() {
|
||||
if (checkTask != null) {
|
||||
checkTask.cancel();
|
||||
}
|
||||
if (scheduledExecutor != null) {
|
||||
scheduledExecutor.shutdownNow();
|
||||
scheduledExecutor = null;
|
||||
}
|
||||
if (workerExecutor != null) {
|
||||
workerExecutor.shutdownNow();
|
||||
workerExecutor = null;
|
||||
}
|
||||
checkTask = null;
|
||||
}
|
||||
|
||||
private ScheduledExecutorService createScheduler() {
|
||||
@@ -178,8 +222,6 @@ public class AlarmGroupReduce implements DisposableBean {
|
||||
groupCacheMap.forEach((groupKey, cache) -> {
|
||||
if (shouldSendGroup(cache, now)) {
|
||||
sendGroupAlert(cache);
|
||||
cache.setLastSendTime(now);
|
||||
cache.getAlertFingerprints().clear();
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
@@ -276,43 +318,50 @@ public class AlarmGroupReduce implements DisposableBean {
|
||||
|
||||
if (shouldSendGroupImmediately(cache)) {
|
||||
sendGroupAlert(cache);
|
||||
cache.setLastSendTime(System.currentTimeMillis());
|
||||
cache.getAlertFingerprints().clear();
|
||||
}
|
||||
}
|
||||
|
||||
private void sendGroupAlert(GroupAlertCache cache) {
|
||||
if (cache.getAlertFingerprints().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
long now = System.currentTimeMillis();
|
||||
String status = determineGroupStatus(cache.getAlertFingerprints().values());
|
||||
|
||||
// For firing alerts, check repeat interval
|
||||
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
|
||||
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
|
||||
long repeatInterval = ruleConfig.getRepeatInterval() != null
|
||||
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
|
||||
|
||||
// Skip if within repeat interval
|
||||
if (cache.getLastRepeatTime() > 0
|
||||
&& now - cache.getLastRepeatTime() < repeatInterval) {
|
||||
synchronized (cache) {
|
||||
Map<String, SingleAlert> snapshot = new HashMap<>(cache.getAlertFingerprints());
|
||||
if (snapshot.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
cache.setLastRepeatTime(now);
|
||||
|
||||
long now = System.currentTimeMillis();
|
||||
String status = determineGroupStatus(snapshot.values());
|
||||
|
||||
// For firing alerts, check repeat interval without consuming the retained snapshot.
|
||||
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
|
||||
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
|
||||
long repeatInterval = ruleConfig.getRepeatInterval() != null
|
||||
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
|
||||
|
||||
if (cache.getLastRepeatTime() > 0
|
||||
&& now - cache.getLastRepeatTime() < repeatInterval) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
GroupAlert groupAlert = GroupAlert.builder()
|
||||
.groupKey(cache.getGroupKey())
|
||||
.groupLabels(cache.getGroupLabels())
|
||||
.commonLabels(extractCommonLabels(snapshot.values()))
|
||||
.commonAnnotations(extractCommonAnnotations(snapshot.values()))
|
||||
.alerts(new ArrayList<>(snapshot.values()))
|
||||
.status(status)
|
||||
.build();
|
||||
|
||||
if (!alarmInhibitReduce.inhibitAlarm(groupAlert)) {
|
||||
return;
|
||||
}
|
||||
snapshot.forEach((fingerprint, alert) ->
|
||||
cache.getAlertFingerprints().remove(fingerprint, alert));
|
||||
cache.setLastSendTime(now);
|
||||
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
|
||||
cache.setLastRepeatTime(now);
|
||||
}
|
||||
}
|
||||
|
||||
GroupAlert groupAlert = GroupAlert.builder()
|
||||
.groupKey(cache.getGroupKey())
|
||||
.groupLabels(cache.getGroupLabels())
|
||||
.commonLabels(extractCommonLabels(cache.getAlertFingerprints().values()))
|
||||
.commonAnnotations(extractCommonAnnotations(cache.getAlertFingerprints().values()))
|
||||
.alerts(new ArrayList<>(cache.getAlertFingerprints().values()))
|
||||
.status(status)
|
||||
.build();
|
||||
|
||||
alarmInhibitReduce.inhibitAlarm(groupAlert);
|
||||
}
|
||||
|
||||
private boolean shouldSendGroup(GroupAlertCache cache, long now) {
|
||||
@@ -414,6 +463,12 @@ public class AlarmGroupReduce implements DisposableBean {
|
||||
|
||||
private int pendingRuns;
|
||||
|
||||
private boolean cancelled;
|
||||
|
||||
private boolean paused;
|
||||
|
||||
private boolean missedWhilePaused;
|
||||
|
||||
private ScheduledDispatchTask(ExecutorService executor, Runnable task) {
|
||||
this.executor = executor;
|
||||
this.task = task;
|
||||
@@ -422,6 +477,13 @@ public class AlarmGroupReduce implements DisposableBean {
|
||||
private void dispatch() {
|
||||
boolean shouldSchedule;
|
||||
synchronized (this) {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
if (paused) {
|
||||
missedWhilePaused = true;
|
||||
return;
|
||||
}
|
||||
pendingRuns++;
|
||||
shouldSchedule = !running;
|
||||
if (shouldSchedule) {
|
||||
@@ -452,14 +514,69 @@ public class AlarmGroupReduce implements DisposableBean {
|
||||
private void scheduleNextIfNeeded() {
|
||||
boolean shouldSchedule;
|
||||
synchronized (this) {
|
||||
if (cancelled) {
|
||||
pendingRuns = 0;
|
||||
running = false;
|
||||
notifyAll();
|
||||
return;
|
||||
}
|
||||
pendingRuns = Math.max(0, pendingRuns - 1);
|
||||
shouldSchedule = pendingRuns > 0;
|
||||
if (!shouldSchedule) {
|
||||
running = false;
|
||||
notifyAll();
|
||||
return;
|
||||
}
|
||||
}
|
||||
scheduleRun();
|
||||
}
|
||||
|
||||
private synchronized void cancel() {
|
||||
cancelled = true;
|
||||
pendingRuns = 0;
|
||||
missedWhilePaused = false;
|
||||
notifyAll();
|
||||
}
|
||||
|
||||
private synchronized void pauseAdmission() {
|
||||
paused = true;
|
||||
missedWhilePaused |= pendingRuns > 1;
|
||||
pendingRuns = running ? 1 : 0;
|
||||
}
|
||||
|
||||
private synchronized void awaitDrained(long timeoutNanos)
|
||||
throws InterruptedException, TimeoutException {
|
||||
long remainingNanos = timeoutNanos;
|
||||
long startedNanos = System.nanoTime();
|
||||
while (running) {
|
||||
if (remainingNanos <= 0) {
|
||||
throw new TimeoutException();
|
||||
}
|
||||
TimeUnit.NANOSECONDS.timedWait(this, remainingNanos);
|
||||
long elapsedNanos = System.nanoTime() - startedNanos;
|
||||
if (elapsedNanos <= 0) {
|
||||
remainingNanos = timeoutNanos;
|
||||
} else if (elapsedNanos >= timeoutNanos) {
|
||||
remainingNanos = 0;
|
||||
} else {
|
||||
remainingNanos = timeoutNanos - elapsedNanos;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void resumeAdmission() {
|
||||
boolean dispatchMissed;
|
||||
synchronized (this) {
|
||||
if (!paused) {
|
||||
return;
|
||||
}
|
||||
paused = false;
|
||||
dispatchMissed = missedWhilePaused && !cancelled;
|
||||
missedWhilePaused = false;
|
||||
}
|
||||
if (dispatchMissed) {
|
||||
dispatch();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+60
-32
@@ -78,29 +78,26 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
*/
|
||||
private final long sourceAlertTtl;
|
||||
|
||||
private final ScheduledExecutorService cleanupScheduler;
|
||||
private final AlertInhibitDao alertInhibitDao;
|
||||
private final VirtualThreadProperties virtualThreadProperties;
|
||||
private ScheduledExecutorService cleanupScheduler;
|
||||
|
||||
private final ExecutorService cleanupExecutor;
|
||||
private ExecutorService cleanupExecutor;
|
||||
|
||||
private final ScheduledDispatchTask cleanupTask;
|
||||
private ScheduledDispatchTask cleanupTask;
|
||||
|
||||
public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao
|
||||
, AlerterProperties alerterProperties) {
|
||||
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults(), true);
|
||||
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, VirtualThreadProperties.defaults());
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
|
||||
AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties) {
|
||||
this(alarmSilenceReduce, alertInhibitDao, alerterProperties, virtualThreadProperties, true);
|
||||
}
|
||||
|
||||
AlarmInhibitReduce(AlarmSilenceReduce alarmSilenceReduce, AlertInhibitDao alertInhibitDao,
|
||||
AlerterProperties alerterProperties, VirtualThreadProperties virtualThreadProperties,
|
||||
boolean autoStart) {
|
||||
this.alarmSilenceReduce = alarmSilenceReduce;
|
||||
VirtualThreadProperties properties =
|
||||
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
this.alertInhibitDao = alertInhibitDao;
|
||||
this.virtualThreadProperties = virtualThreadProperties == null
|
||||
? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
if (alerterProperties.getInhibit() != null && alerterProperties.getInhibit().getTtl() > 0) {
|
||||
this.sourceAlertTtl = alerterProperties.getInhibit().getTtl();
|
||||
} else {
|
||||
@@ -108,34 +105,49 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
}
|
||||
inhibitRules = new ConcurrentHashMap<>(8);
|
||||
sourceAlertCache = new ConcurrentHashMap<>(8);
|
||||
this.cleanupScheduler = createCleanupScheduler();
|
||||
this.cleanupExecutor = createCleanupExecutor(properties);
|
||||
this.cleanupTask = new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache);
|
||||
List<AlertInhibit> inhibits = alertInhibitDao.findAlertInhibitsByEnableIsTrue();
|
||||
refreshInhibitRules(inhibits);
|
||||
if (autoStart) {
|
||||
startScheduledCleanupCache();
|
||||
}
|
||||
}
|
||||
|
||||
private void startScheduledCleanupCache() {
|
||||
cleanupScheduler.scheduleAtFixedRate(this::dispatchCleanupCache, CHECK_INTERVAL, CHECK_INTERVAL,
|
||||
synchronized void start() {
|
||||
if (cleanupScheduler != null) {
|
||||
return;
|
||||
}
|
||||
cleanupScheduler = createCleanupScheduler();
|
||||
cleanupExecutor = createCleanupExecutor(virtualThreadProperties);
|
||||
ScheduledDispatchTask currentCleanupTask =
|
||||
new ScheduledDispatchTask(cleanupExecutor, this::runCleanupCache);
|
||||
cleanupTask = currentCleanupTask;
|
||||
refreshInhibitRules(alertInhibitDao.findAlertInhibitsByEnableIsTrue());
|
||||
startScheduledCleanupCache(currentCleanupTask);
|
||||
}
|
||||
|
||||
private void startScheduledCleanupCache(ScheduledDispatchTask currentCleanupTask) {
|
||||
cleanupScheduler.scheduleAtFixedRate(currentCleanupTask::dispatch, CHECK_INTERVAL, CHECK_INTERVAL,
|
||||
TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
void dispatchCleanupCache() {
|
||||
cleanupTask.dispatch();
|
||||
synchronized void dispatchCleanupCache() {
|
||||
if (cleanupTask != null) {
|
||||
cleanupTask.dispatch();
|
||||
}
|
||||
}
|
||||
|
||||
void beforeCleanupCacheRun() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
cleanupScheduler.shutdownNow();
|
||||
public synchronized void destroy() {
|
||||
if (cleanupTask != null) {
|
||||
cleanupTask.cancel();
|
||||
}
|
||||
if (cleanupScheduler != null) {
|
||||
cleanupScheduler.shutdownNow();
|
||||
cleanupScheduler = null;
|
||||
}
|
||||
if (cleanupExecutor != null) {
|
||||
cleanupExecutor.shutdownNow();
|
||||
cleanupExecutor = null;
|
||||
}
|
||||
cleanupTask = null;
|
||||
}
|
||||
|
||||
private ScheduledExecutorService createCleanupScheduler() {
|
||||
@@ -191,15 +203,14 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
* If alert is inhibited, it will not be forwarded
|
||||
* @param groupAlert Grouped and pending alerts to be processed
|
||||
*/
|
||||
public void inhibitAlarm(GroupAlert groupAlert) {
|
||||
public boolean inhibitAlarm(GroupAlert groupAlert) {
|
||||
if (groupAlert == null) {
|
||||
log.warn("Received null GroupAlert. Skipping processing.");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
if (inhibitRules.isEmpty()) {
|
||||
alarmSilenceReduce.silenceAlarm(groupAlert);
|
||||
return;
|
||||
return alarmSilenceReduce.silenceAlarm(groupAlert);
|
||||
}
|
||||
|
||||
// Process each individual alert
|
||||
@@ -216,10 +227,12 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
|
||||
// Continue processing if there are remaining alerts
|
||||
if (!groupAlert.getAlerts().isEmpty()) {
|
||||
alarmSilenceReduce.silenceAlarm(groupAlert);
|
||||
return alarmSilenceReduce.silenceAlarm(groupAlert);
|
||||
}
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
log.error("Error inhibiting alarm for {}", groupAlert, e);
|
||||
log.error("Alarm inhibit metadata processing failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -389,6 +402,8 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
|
||||
private int pendingRuns;
|
||||
|
||||
private boolean cancelled;
|
||||
|
||||
private ScheduledDispatchTask(ExecutorService executor, Runnable task) {
|
||||
this.executor = executor;
|
||||
this.task = task;
|
||||
@@ -397,6 +412,9 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
private void dispatch() {
|
||||
boolean shouldSchedule;
|
||||
synchronized (this) {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
pendingRuns++;
|
||||
shouldSchedule = !running;
|
||||
if (shouldSchedule) {
|
||||
@@ -427,6 +445,11 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
private void scheduleNextIfNeeded() {
|
||||
boolean shouldSchedule;
|
||||
synchronized (this) {
|
||||
if (cancelled) {
|
||||
pendingRuns = 0;
|
||||
running = false;
|
||||
return;
|
||||
}
|
||||
pendingRuns = Math.max(0, pendingRuns - 1);
|
||||
shouldSchedule = pendingRuns > 0;
|
||||
if (!shouldSchedule) {
|
||||
@@ -436,5 +459,10 @@ public class AlarmInhibitReduce implements DisposableBean {
|
||||
}
|
||||
scheduleRun();
|
||||
}
|
||||
|
||||
private synchronized void cancel() {
|
||||
cancelled = true;
|
||||
pendingRuns = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.alert.reduce;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Loads reducer rules and starts cleanup/grouping workers only in normal runtime. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class AlarmReduceLifecycle implements CommandLineRunner {
|
||||
|
||||
private final AlarmInhibitReduce inhibitReduce;
|
||||
private final AlarmGroupReduce groupReduce;
|
||||
|
||||
public AlarmReduceLifecycle(AlarmInhibitReduce inhibitReduce, AlarmGroupReduce groupReduce) {
|
||||
this.inhibitReduce = inhibitReduce;
|
||||
this.groupReduce = groupReduce;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
inhibitReduce.start();
|
||||
groupReduce.start();
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -46,7 +46,7 @@ public class AlarmSilenceReduce {
|
||||
* If alert matches any active silence rule, it will be silenced
|
||||
* @param groupAlert The alert to be processed
|
||||
*/
|
||||
public void silenceAlarm(GroupAlert groupAlert) {
|
||||
public boolean silenceAlarm(GroupAlert groupAlert) {
|
||||
List<AlertSilence> alertSilenceList = CacheFactory.getAlertSilenceCache();
|
||||
if (alertSilenceList == null) {
|
||||
alertSilenceList = alertSilenceDao.findAlertSilencesByEnableTrue();
|
||||
@@ -72,21 +72,21 @@ public class AlarmSilenceReduce {
|
||||
continue;
|
||||
}
|
||||
// Alert is silenced
|
||||
return;
|
||||
return true;
|
||||
} else if (alertSilence.getType() == 1) {
|
||||
// Cyclic silence rule
|
||||
int currentDayOfWeek = now.getDayOfWeek().getValue();
|
||||
if (alertSilence.getDays() != null && alertSilence.getDays().contains((byte) currentDayOfWeek)
|
||||
&& !checkAndSave(now, alertSilence)) {
|
||||
// Alert is silenced
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No matching silence rule, forward the alert
|
||||
dispatcherAlarm.dispatchAlarm(groupAlert);
|
||||
return dispatcherAlarm.dispatchAlarm(groupAlert);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+2
-4
@@ -34,7 +34,6 @@ import org.apache.hertzbeat.alert.service.NoticeTemplateMutationException;
|
||||
import org.apache.hertzbeat.alert.service.NoticeConfigService;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
@@ -69,7 +68,7 @@ import java.util.stream.Collectors;
|
||||
@Order(value = Ordered.HIGHEST_PRECEDENCE)
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Slf4j
|
||||
public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLineRunner {
|
||||
public class NoticeConfigServiceImpl implements NoticeConfigService {
|
||||
|
||||
private static final Map<Byte, NoticeTemplate> PRESET_TEMPLATE = new HashMap<>(16);
|
||||
|
||||
@@ -372,8 +371,7 @@ public class NoticeConfigServiceImpl implements NoticeConfigService, CommandLine
|
||||
CacheFactory.clearNoticeCache();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) throws Exception {
|
||||
void loadPresetTemplates() {
|
||||
try {
|
||||
log.info("load default notice template in internal jar");
|
||||
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.alert.service.impl;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Loads preset templates only after the business runtime is opened. */
|
||||
@Component
|
||||
@Order(Ordered.HIGHEST_PRECEDENCE)
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class NoticeTemplateInitializer implements CommandLineRunner {
|
||||
|
||||
private final NoticeConfigServiceImpl noticeConfigService;
|
||||
|
||||
public NoticeTemplateInitializer(NoticeConfigServiceImpl noticeConfigService) {
|
||||
this.noticeConfigService = noticeConfigService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
noticeConfigService.loadPresetTemplates();
|
||||
}
|
||||
}
|
||||
+121
-5
@@ -20,14 +20,18 @@ package org.apache.hertzbeat.alert.calculate.periodic;
|
||||
import static org.apache.hertzbeat.common.constants.CommonConstants.METRIC_ALERT_THRESHOLD_TYPE_PERIODIC;
|
||||
import static org.apache.hertzbeat.common.constants.CommonConstants.TRACE_ALERT_THRESHOLD_TYPE_PERIODIC;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.clearInvocations;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.apache.hertzbeat.alert.dao.AlertDefineDao;
|
||||
@@ -64,12 +68,13 @@ class PeriodicAlertRuleSchedulerTest {
|
||||
void setUp() {
|
||||
scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, traceCalculator, alertDefineDao,
|
||||
VirtualThreadProperties.defaults());
|
||||
scheduler.start();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
if (scheduler != null) {
|
||||
scheduler.destroy();
|
||||
scheduler.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,9 +156,10 @@ class PeriodicAlertRuleSchedulerTest {
|
||||
|
||||
@Test
|
||||
void updateScheduleHonorsConfiguredGlobalPeriodicConcurrencyLimit() throws InterruptedException {
|
||||
scheduler.destroy();
|
||||
scheduler.stop();
|
||||
scheduler = new PeriodicAlertRuleScheduler(metricsCalculator, logCalculator, traceCalculator, alertDefineDao,
|
||||
periodicProperties(1));
|
||||
scheduler.start();
|
||||
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
@@ -191,7 +197,9 @@ class PeriodicAlertRuleSchedulerTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void runLoadsPeriodicTraceRulesAtStartup() throws Exception {
|
||||
void startLoadsPeriodicTraceRulesAtStartup() {
|
||||
scheduler.stop();
|
||||
clearInvocations(alertDefineDao);
|
||||
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(METRIC_ALERT_THRESHOLD_TYPE_PERIODIC))
|
||||
.thenReturn(java.util.List.of());
|
||||
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(
|
||||
@@ -200,7 +208,7 @@ class PeriodicAlertRuleSchedulerTest {
|
||||
when(alertDefineDao.findAlertDefinesByTypeAndEnableTrue(TRACE_ALERT_THRESHOLD_TYPE_PERIODIC))
|
||||
.thenReturn(java.util.List.of(traceRule(6L)));
|
||||
|
||||
scheduler.run();
|
||||
scheduler.start();
|
||||
|
||||
verify(alertDefineDao).findAlertDefinesByTypeAndEnableTrue(TRACE_ALERT_THRESHOLD_TYPE_PERIODIC);
|
||||
}
|
||||
@@ -218,6 +226,110 @@ class PeriodicAlertRuleSchedulerTest {
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void stopDuringPendingExecutionIsIdempotentAndDoesNotResubmit() throws InterruptedException {
|
||||
CountDownLatch started = new CountDownLatch(1);
|
||||
CountDownLatch interrupted = new CountDownLatch(1);
|
||||
CountDownLatch secondStarted = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
int current = invocations.incrementAndGet();
|
||||
if (current == 1) {
|
||||
started.countDown();
|
||||
try {
|
||||
Thread.sleep(5000L);
|
||||
} catch (InterruptedException e) {
|
||||
interrupted.countDown();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
} else {
|
||||
secondStarted.countDown();
|
||||
}
|
||||
return null;
|
||||
}).when(metricsCalculator).calculate(any(AlertDefine.class));
|
||||
|
||||
scheduler.updateSchedule(metricRule(8L));
|
||||
assertTrue(started.await(5, TimeUnit.SECONDS));
|
||||
Thread.sleep(1200L);
|
||||
|
||||
scheduler.stop();
|
||||
scheduler.stop();
|
||||
|
||||
assertTrue(interrupted.await(5, TimeUnit.SECONDS));
|
||||
assertFalse(secondStarted.await(1500, TimeUnit.MILLISECONDS));
|
||||
assertEquals(1, invocations.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseDrainsAnEnteredPeriodicCalculationWithoutStoppingScheduler() throws Exception {
|
||||
CountDownLatch started = new CountDownLatch(1);
|
||||
CountDownLatch release = new CountDownLatch(1);
|
||||
CountDownLatch resumed = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
if (invocations.incrementAndGet() == 1) {
|
||||
started.countDown();
|
||||
release.await();
|
||||
} else {
|
||||
resumed.countDown();
|
||||
}
|
||||
return null;
|
||||
}).when(metricsCalculator).calculate(any(AlertDefine.class));
|
||||
scheduler.updateSchedule(metricRule(9L));
|
||||
assertTrue(started.await(5, TimeUnit.SECONDS));
|
||||
|
||||
scheduler.pauseAdmission();
|
||||
assertThrows(TimeoutException.class, () -> scheduler.awaitDrained(0));
|
||||
release.countDown();
|
||||
scheduler.awaitDrained(TimeUnit.SECONDS.toNanos(1));
|
||||
|
||||
scheduler.resumeAdmission();
|
||||
scheduler.updateSchedule(metricRule(10L));
|
||||
assertTrue(resumed.await(5, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pausedTicksCoalescePerRuleAndResumeOnceWithVirtualExecutor() throws Exception {
|
||||
assertPausedTicksResumeOnce(periodicProperties(true, 1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pausedTicksCoalescePerRuleAndResumeOnceWithScheduledExecutor() throws Exception {
|
||||
assertPausedTicksResumeOnce(periodicProperties(false, 1));
|
||||
}
|
||||
|
||||
private void assertPausedTicksResumeOnce(VirtualThreadProperties properties) throws Exception {
|
||||
scheduler.stop();
|
||||
CountDownLatch pausedTicks = new CountDownLatch(2);
|
||||
scheduler = new PeriodicAlertRuleScheduler(
|
||||
metricsCalculator, logCalculator, traceCalculator, alertDefineDao, properties) {
|
||||
@Override
|
||||
void beforeRuleTrigger(AlertDefine rule) {
|
||||
pausedTicks.countDown();
|
||||
}
|
||||
};
|
||||
scheduler.start();
|
||||
CountDownLatch calculated = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
invocations.incrementAndGet();
|
||||
calculated.countDown();
|
||||
return null;
|
||||
}).when(metricsCalculator).calculate(any(AlertDefine.class));
|
||||
AlertDefine rule = metricRule(11L);
|
||||
|
||||
scheduler.pauseAdmission();
|
||||
scheduler.updateSchedule(rule);
|
||||
assertTrue(pausedTicks.await(3, TimeUnit.SECONDS));
|
||||
scheduler.awaitDrained(0);
|
||||
|
||||
scheduler.resumeAdmission();
|
||||
scheduler.resumeAdmission();
|
||||
assertTrue(calculated.await(1, TimeUnit.SECONDS));
|
||||
scheduler.cancelSchedule(rule.getId());
|
||||
assertEquals(1, invocations.get());
|
||||
}
|
||||
|
||||
private AlertDefine metricRule(Long id) {
|
||||
return AlertDefine.builder()
|
||||
.id(id)
|
||||
@@ -239,8 +351,12 @@ class PeriodicAlertRuleSchedulerTest {
|
||||
}
|
||||
|
||||
private VirtualThreadProperties periodicProperties(int maxConcurrentJobs) {
|
||||
return periodicProperties(true, maxConcurrentJobs);
|
||||
}
|
||||
|
||||
private VirtualThreadProperties periodicProperties(boolean enabled, int maxConcurrentJobs) {
|
||||
return new VirtualThreadProperties(
|
||||
true,
|
||||
enabled,
|
||||
VirtualThreadProperties.PoolProperties.collectorDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.commonDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.managerDefaults(),
|
||||
|
||||
+63
@@ -23,7 +23,9 @@ import org.apache.hertzbeat.alert.calculate.AlarmCacheManager;
|
||||
import org.apache.hertzbeat.alert.calculate.JexlExprCalculator;
|
||||
import org.apache.hertzbeat.alert.dao.SingleAlertDao;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce;
|
||||
import org.apache.hertzbeat.alert.service.AlertDefineService;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
@@ -40,8 +42,12 @@ import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
@@ -71,6 +77,62 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
|
||||
|
||||
private MetricsRealTimeAlertCalculator metricsRealTimeAlertCalculator;
|
||||
|
||||
@Test
|
||||
void positiveCapacityMaintenanceBufferKeepsTelemetryLoopForwardingLaterSamples() throws Exception {
|
||||
int sampleCount = 8;
|
||||
CountDownLatch stored = new CountDownLatch(sampleCount);
|
||||
InMemoryCommonDataQueue queue = new InMemoryCommonDataQueue() {
|
||||
@Override
|
||||
public void sendMetricsDataToStorage(CollectRep.MetricsData metricsData) {
|
||||
super.sendMetricsDataToStorage(metricsData);
|
||||
stored.countDown();
|
||||
}
|
||||
};
|
||||
AlarmCommonReduce reduce = new AlarmCommonReduce(
|
||||
org.mockito.Mockito.mock(AlarmGroupReduce.class), positiveReduceCapacityProperties());
|
||||
AlerterWorkerPool loopPool = new AlerterWorkerPool();
|
||||
MetricsRealTimeAlertCalculator calculator = new MetricsRealTimeAlertCalculator(
|
||||
loopPool, queue, alertDefineService, singleAlertDao, reduce, alarmCacheManager,
|
||||
new JexlExprCalculator(), false) {
|
||||
@Override
|
||||
protected void calculate(CollectRep.MetricsData metricsData) {
|
||||
reduce.reduceAndSendAlarm(org.apache.hertzbeat.common.entity.alerter.SingleAlert.builder()
|
||||
.labels(Map.of("sample", Long.toString(metricsData.getId())))
|
||||
.build());
|
||||
}
|
||||
};
|
||||
reduce.pauseAdmission();
|
||||
calculator.startCalculate();
|
||||
for (int index = 0; index < sampleCount; index++) {
|
||||
queue.sendMetricsData(CollectRep.MetricsData.newBuilder().setId(index + 1L).build());
|
||||
}
|
||||
|
||||
assertTrue(stored.await(2, TimeUnit.SECONDS));
|
||||
for (int index = 0; index < sampleCount; index++) {
|
||||
assertNotNull(queue.pollMetricsDataToStorage());
|
||||
}
|
||||
|
||||
reduce.destroy();
|
||||
loopPool.destroy();
|
||||
}
|
||||
|
||||
private static VirtualThreadProperties positiveReduceCapacityProperties() {
|
||||
return new VirtualThreadProperties(
|
||||
true,
|
||||
VirtualThreadProperties.PoolProperties.collectorDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.commonDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.managerDefaults(),
|
||||
new VirtualThreadProperties.AlerterProperties(
|
||||
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
|
||||
10,
|
||||
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
|
||||
new VirtualThreadProperties.QueueProperties(1, 1),
|
||||
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
|
||||
4),
|
||||
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
|
||||
VirtualThreadProperties.AsyncProperties.defaults());
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
@@ -169,6 +231,7 @@ public class MetricsRealTimeAlertCalculatorMatchTest {
|
||||
verify(alarmCacheManager, times(1)).getPending(any(), any());
|
||||
verify(alarmCacheManager, times(1)).putFiring(any(), any(), any());
|
||||
verify(alarmCommonReduce, times(1)).reduceAndSendAlarm(any());
|
||||
verify(dataQueue, times(1)).sendMetricsDataToStorage(metricsData);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+16
@@ -24,6 +24,8 @@ import static org.mockito.ArgumentMatchers.anyByte;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doNothing;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
@@ -180,4 +182,18 @@ class AlertNoticeDispatchTest {
|
||||
verify(alertNotifyHandler).send(eq(receiver), eq(template), eq(alert));
|
||||
verify(emitterManager).broadcast(any(String.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void postStoreFailureDoesNotChangeMetadataSuccessOutcome() {
|
||||
when(alertStoreHandler.store(alert)).thenReturn(alert);
|
||||
when(noticeConfigService.getReceiverFilterRule(alert))
|
||||
.thenThrow(new IllegalStateException("notice unavailable"));
|
||||
doThrow(new IllegalStateException("broadcast unavailable"))
|
||||
.when(emitterManager).broadcast(any(String.class));
|
||||
|
||||
assertTrue(alertNoticeDispatch.dispatchAlarm(alert));
|
||||
|
||||
verify(alertStoreHandler, times(1)).store(alert);
|
||||
verify(emitterManager).broadcast(any(String.class));
|
||||
}
|
||||
}
|
||||
|
||||
+30
-1
@@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.hertzbeat.alert.notice.impl;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
@@ -46,6 +47,8 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.ResourceBundle;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
/**
|
||||
* Test case for Email Alert Notify
|
||||
@@ -106,7 +109,7 @@ class EmailAlertNotifyHandlerImplTest {
|
||||
.content(JsonUtil.toJson(mailServerConfig))
|
||||
.build();
|
||||
when(generalConfigDao.findByType(any())).thenReturn(generalConfig);
|
||||
when(mailSender.getJavaMailProperties()).thenReturn(new Properties());
|
||||
lenient().when(mailSender.getJavaMailProperties()).thenReturn(new Properties());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -130,4 +133,30 @@ class EmailAlertNotifyHandlerImplTest {
|
||||
assertThrows(AlertNoticeException.class,
|
||||
() -> emailAlertNotifyHandler.send(receiver, template, groupAlert));
|
||||
}
|
||||
|
||||
@Test
|
||||
void configuredFromAddressIsUsedByProductionHandler() throws Exception {
|
||||
AtomicReference<MimeMessage> sent = new AtomicReference<>();
|
||||
JavaMailSenderImpl sender = new JavaMailSenderImpl() {
|
||||
@Override
|
||||
public void send(MimeMessage mimeMessage) {
|
||||
sent.set(mimeMessage);
|
||||
}
|
||||
};
|
||||
when(generalConfigDao.findByType(any())).thenReturn(null);
|
||||
EmailAlertNotifyHandlerImpl handler = new EmailAlertNotifyHandlerImpl(sender, generalConfigDao);
|
||||
ReflectionTestUtils.setField(handler, "host", "smtp.example.test");
|
||||
ReflectionTestUtils.setField(handler, "port", 465);
|
||||
ReflectionTestUtils.setField(handler, "username", "smtp-user@example.test");
|
||||
ReflectionTestUtils.setField(handler, "password", "password");
|
||||
ReflectionTestUtils.setField(handler, "fromAddress", "alerts@example.test");
|
||||
ReflectionTestUtils.setField(handler, "sslEnable", true);
|
||||
ReflectionTestUtils.setField(handler, "starttlsEnable", false);
|
||||
ReflectionTestUtils.setField(handler, "bundle", bundle);
|
||||
when(bundle.getString("alerter.notify.title")).thenReturn("Alert Notification");
|
||||
|
||||
handler.send(receiver, template, groupAlert);
|
||||
|
||||
assertEquals("alerts@example.test", sent.get().getFrom()[0].toString());
|
||||
}
|
||||
}
|
||||
|
||||
+210
@@ -18,20 +18,26 @@
|
||||
package org.apache.hertzbeat.alert.reduce;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.timeout;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -145,4 +151,208 @@ class AlarmCommonReduceTest {
|
||||
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseDefersNewReducersAndReplaysEachOnceAfterDrain() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch resumedStarted = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
int current = invocations.incrementAndGet();
|
||||
if (current == 1) {
|
||||
firstStarted.countDown();
|
||||
releaseFirst.await();
|
||||
} else {
|
||||
resumedStarted.countDown();
|
||||
}
|
||||
return null;
|
||||
}).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class));
|
||||
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
|
||||
alarmCommonReduce.pauseAdmission();
|
||||
assertThrows(TimeoutException.class, () -> alarmCommonReduce.awaitDrained(0));
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
releaseFirst.countDown();
|
||||
alarmCommonReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
|
||||
verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class));
|
||||
|
||||
alarmCommonReduce.resumeAdmission();
|
||||
assertTrue(resumedStarted.await(5, TimeUnit.SECONDS));
|
||||
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseCannotLetDeferredWaiterOvertakeReservedSubmission() throws Exception {
|
||||
CountDownLatch permitReserved = new CountDownLatch(1);
|
||||
CountDownLatch releaseSubmission = new CountDownLatch(1);
|
||||
alarmCommonReduce.destroy();
|
||||
alarmCommonReduce = new TestAlarmCommonReduce(
|
||||
alarmGroupReduce, permitReserved, releaseSubmission);
|
||||
CountDownLatch firstRunning = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch secondRunning = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
doAnswer(invocation -> {
|
||||
if (invocations.incrementAndGet() == 1) {
|
||||
firstRunning.countDown();
|
||||
releaseFirst.await();
|
||||
} else {
|
||||
secondRunning.countDown();
|
||||
}
|
||||
return null;
|
||||
}).when(alarmGroupReduce).processGroupAlert(any(SingleAlert.class));
|
||||
|
||||
Thread admitted = Thread.ofPlatform().start(() -> alarmCommonReduce.reduceAndSendAlarm(testAlert));
|
||||
assertTrue(permitReserved.await(1, TimeUnit.SECONDS));
|
||||
Thread pause = Thread.ofPlatform().start(alarmCommonReduce::pauseAdmission);
|
||||
while (!alarmCommonReduce.hasQueuedMaintenanceThread(pause)) {
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
Thread deferred = Thread.ofPlatform().start(() -> alarmCommonReduce.reduceAndSendAlarm(testAlert));
|
||||
releaseSubmission.countDown();
|
||||
admitted.join(1_000);
|
||||
pause.join(1_000);
|
||||
deferred.join(1_000);
|
||||
assertTrue(firstRunning.await(1, TimeUnit.SECONDS));
|
||||
|
||||
assertThrows(TimeoutException.class, () -> alarmCommonReduce.awaitDrained(0));
|
||||
releaseFirst.countDown();
|
||||
alarmCommonReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
|
||||
verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class));
|
||||
|
||||
alarmCommonReduce.resumeAdmission();
|
||||
assertTrue(secondRunning.await(1, TimeUnit.SECONDS));
|
||||
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void maintenanceDeferralNeverBlocksProducerAtWorkerQueueCapacity() throws Exception {
|
||||
ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class);
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
((Runnable) invocation.getArgument(0)).run();
|
||||
return null;
|
||||
}).when(executor).execute(any(Runnable.class));
|
||||
alarmCommonReduce.destroy();
|
||||
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor);
|
||||
alarmCommonReduce.pauseAdmission();
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
CountDownLatch callerContinued = new CountDownLatch(1);
|
||||
Thread second = Thread.ofPlatform().start(() -> {
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
callerContinued.countDown();
|
||||
});
|
||||
assertTrue(callerContinued.await(1, TimeUnit.SECONDS));
|
||||
second.join(1_000);
|
||||
|
||||
verify(executor, times(0)).execute(any(Runnable.class));
|
||||
assertEquals(2, alarmCommonReduce.deferredTaskCount());
|
||||
|
||||
alarmCommonReduce.resumeAdmission();
|
||||
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void replayQueueRejectionRetainsEveryUnsubmittedDeferredTask() {
|
||||
ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class);
|
||||
AtomicInteger submissions = new AtomicInteger();
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
if (submissions.incrementAndGet() == 2) {
|
||||
throw new RejectedExecutionException();
|
||||
}
|
||||
((Runnable) invocation.getArgument(0)).run();
|
||||
return null;
|
||||
}).when(executor).execute(any(Runnable.class));
|
||||
alarmCommonReduce.destroy();
|
||||
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor);
|
||||
alarmCommonReduce.pauseAdmission();
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
|
||||
assertThrows(RejectedExecutionException.class, alarmCommonReduce::resumeAdmission);
|
||||
assertEquals(1, alarmCommonReduce.deferredTaskCount());
|
||||
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
((Runnable) invocation.getArgument(0)).run();
|
||||
return null;
|
||||
}).when(executor).execute(any(Runnable.class));
|
||||
alarmCommonReduce.resumeAdmission();
|
||||
|
||||
assertEquals(0, alarmCommonReduce.deferredTaskCount());
|
||||
verify(alarmGroupReduce, times(2)).processGroupAlert(any(SingleAlert.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectedReplayRemainsDeferredForOneRetry() {
|
||||
ManagedExecutor executor = org.mockito.Mockito.mock(ManagedExecutor.class);
|
||||
org.mockito.Mockito.doThrow(new RejectedExecutionException()).when(executor).execute(any(Runnable.class));
|
||||
alarmCommonReduce.destroy();
|
||||
alarmCommonReduce = new AlarmCommonReduce(alarmGroupReduce, executor);
|
||||
alarmCommonReduce.pauseAdmission();
|
||||
alarmCommonReduce.reduceAndSendAlarm(testAlert);
|
||||
|
||||
assertThrows(RejectedExecutionException.class,
|
||||
alarmCommonReduce::resumeAdmission);
|
||||
assertEquals(1, alarmCommonReduce.deferredTaskCount());
|
||||
|
||||
org.mockito.Mockito.doAnswer(invocation -> {
|
||||
((Runnable) invocation.getArgument(0)).run();
|
||||
return null;
|
||||
}).when(executor).execute(any(Runnable.class));
|
||||
alarmCommonReduce.resumeAdmission();
|
||||
|
||||
assertEquals(0, alarmCommonReduce.deferredTaskCount());
|
||||
verify(alarmGroupReduce, times(1)).processGroupAlert(any(SingleAlert.class));
|
||||
}
|
||||
|
||||
private static final class TestAlarmCommonReduce extends AlarmCommonReduce {
|
||||
|
||||
private final CountDownLatch permitReserved;
|
||||
private final CountDownLatch releaseSubmission;
|
||||
private final AtomicBoolean first = new AtomicBoolean(true);
|
||||
|
||||
private TestAlarmCommonReduce(
|
||||
AlarmGroupReduce alarmGroupReduce,
|
||||
CountDownLatch permitReserved,
|
||||
CountDownLatch releaseSubmission) {
|
||||
super(alarmGroupReduce, singleWorkerProperties());
|
||||
this.permitReserved = permitReserved;
|
||||
this.releaseSubmission = releaseSubmission;
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeAdmittedSubmission() {
|
||||
if (first.compareAndSet(true, false)) {
|
||||
permitReserved.countDown();
|
||||
try {
|
||||
releaseSubmission.await();
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static VirtualThreadProperties singleWorkerProperties() {
|
||||
return singleWorkerProperties(1);
|
||||
}
|
||||
|
||||
private static VirtualThreadProperties singleWorkerProperties(int queueCapacity) {
|
||||
return new VirtualThreadProperties(
|
||||
true,
|
||||
VirtualThreadProperties.PoolProperties.collectorDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.commonDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.managerDefaults(),
|
||||
new VirtualThreadProperties.AlerterProperties(
|
||||
VirtualThreadProperties.PoolProperties.alerterNotifyDefaults(),
|
||||
10,
|
||||
VirtualThreadProperties.QueueProperties.logWorkerDefaults(),
|
||||
new VirtualThreadProperties.QueueProperties(1, queueCapacity),
|
||||
VirtualThreadProperties.QueueProperties.windowEvaluatorDefaults(),
|
||||
4),
|
||||
VirtualThreadProperties.PoolProperties.warehouseDefaults(),
|
||||
VirtualThreadProperties.AsyncProperties.defaults());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+154
-2
@@ -40,18 +40,25 @@ package org.apache.hertzbeat.alert.reduce;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.clearInvocations;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao;
|
||||
@@ -69,6 +76,21 @@ import org.mockito.MockitoAnnotations;
|
||||
*/
|
||||
class AlarmGroupReduceTest {
|
||||
|
||||
@Test
|
||||
void constructorIsPassiveAndLifecycleIsIdempotent() {
|
||||
clearInvocations(alertGroupConvergeDao);
|
||||
AlarmGroupReduce inactive = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
|
||||
new VirtualThreadProperties());
|
||||
|
||||
verifyNoInteractions(alertGroupConvergeDao);
|
||||
|
||||
inactive.start();
|
||||
inactive.start();
|
||||
verify(alertGroupConvergeDao, times(1)).findAlertGroupConvergesByEnableIsTrue();
|
||||
inactive.destroy();
|
||||
inactive.destroy();
|
||||
}
|
||||
|
||||
@Mock
|
||||
private AlarmInhibitReduce alarmInhibitReduce;
|
||||
|
||||
@@ -80,10 +102,12 @@ class AlarmGroupReduceTest {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
when(alarmInhibitReduce.inhibitAlarm(any())).thenReturn(true);
|
||||
when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue())
|
||||
.thenReturn(Collections.emptyList());
|
||||
alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
|
||||
new VirtualThreadProperties(), false);
|
||||
new VirtualThreadProperties());
|
||||
alarmGroupReduce.start();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
@@ -136,6 +160,7 @@ class AlarmGroupReduceTest {
|
||||
alarmGroupReduce.destroy();
|
||||
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
|
||||
new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null);
|
||||
alarmGroupReduce.start();
|
||||
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
|
||||
@@ -143,6 +168,13 @@ class AlarmGroupReduceTest {
|
||||
assertTrue(virtualThread.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void dispatchAfterDestroyIsSafeNoOp() {
|
||||
alarmGroupReduce.destroy();
|
||||
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
}
|
||||
|
||||
@Test
|
||||
void dispatchCheckAndSendGroupsDoesNotRunConcurrently() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
@@ -153,6 +185,7 @@ class AlarmGroupReduceTest {
|
||||
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
|
||||
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
|
||||
maxConcurrent, new AtomicInteger());
|
||||
alarmGroupReduce.start();
|
||||
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
@@ -165,6 +198,125 @@ class AlarmGroupReduceTest {
|
||||
assertEquals(1, maxConcurrent.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void destroyWhileCheckIsRunningDropsPendingDispatch() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch secondStarted = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
alarmGroupReduce.destroy();
|
||||
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
|
||||
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
|
||||
new AtomicInteger(), invocations);
|
||||
alarmGroupReduce.start();
|
||||
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
|
||||
alarmGroupReduce.destroy();
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
|
||||
assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS));
|
||||
assertEquals(1, invocations.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void pauseDrainsRunningGroupPassAndCoalescesOneMissedPass() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch resumedStarted = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
alarmGroupReduce.destroy();
|
||||
alarmGroupReduce = new TestAlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
|
||||
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, resumedStarted,
|
||||
new AtomicInteger(), invocations);
|
||||
alarmGroupReduce.start();
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
|
||||
alarmGroupReduce.pauseAdmission();
|
||||
assertThrows(TimeoutException.class, () -> alarmGroupReduce.awaitDrained(0));
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
releaseFirst.countDown();
|
||||
alarmGroupReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
|
||||
|
||||
alarmGroupReduce.resumeAdmission();
|
||||
assertTrue(resumedStarted.await(5, TimeUnit.SECONDS));
|
||||
assertEquals(2, invocations.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void failedGroupStoreRetainsSnapshotForOneRetry() throws Exception {
|
||||
alarmGroupReduce.refreshGroupDefines(List.of(groupRule(0)));
|
||||
when(alarmInhibitReduce.inhibitAlarm(any())).thenReturn(false, true);
|
||||
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
|
||||
|
||||
dispatchAndDrain();
|
||||
dispatchAndDrain();
|
||||
|
||||
verify(alarmInhibitReduce, times(2)).inhibitAlarm(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void repeatSkipRetainsFiringUntilResolvedSnapshotIsStored() throws Exception {
|
||||
AlertGroupConverge rule = groupRule(600);
|
||||
alarmGroupReduce.refreshGroupDefines(List.of(rule));
|
||||
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
|
||||
dispatchAndDrain();
|
||||
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
|
||||
dispatchAndDrain();
|
||||
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "resolved"));
|
||||
dispatchAndDrain();
|
||||
|
||||
verify(alarmInhibitReduce, times(2)).inhibitAlarm(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void concurrentInsertIsNotClearedWithSuccessfulSnapshot() throws Exception {
|
||||
alarmGroupReduce.refreshGroupDefines(List.of(groupRule(0)));
|
||||
AtomicBoolean inserted = new AtomicBoolean();
|
||||
doAnswer(invocation -> {
|
||||
if (inserted.compareAndSet(false, true)) {
|
||||
alarmGroupReduce.processGroupAlert(groupAlert("fp-2", "firing"));
|
||||
}
|
||||
return true;
|
||||
}).when(alarmInhibitReduce).inhibitAlarm(any());
|
||||
alarmGroupReduce.processGroupAlert(groupAlert("fp-1", "firing"));
|
||||
|
||||
dispatchAndDrain();
|
||||
dispatchAndDrain();
|
||||
dispatchAndDrain();
|
||||
|
||||
verify(alarmInhibitReduce, times(2)).inhibitAlarm(any());
|
||||
}
|
||||
|
||||
private void dispatchAndDrain() throws Exception {
|
||||
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||
alarmGroupReduce.pauseAdmission();
|
||||
alarmGroupReduce.awaitDrained(TimeUnit.SECONDS.toNanos(1));
|
||||
alarmGroupReduce.resumeAdmission();
|
||||
}
|
||||
|
||||
private AlertGroupConverge groupRule(long repeatInterval) {
|
||||
AlertGroupConverge rule = new AlertGroupConverge();
|
||||
rule.setName("test-rule");
|
||||
rule.setGroupLabels(List.of("severity"));
|
||||
rule.setGroupWait(0L);
|
||||
rule.setGroupInterval(0L);
|
||||
rule.setRepeatInterval(repeatInterval);
|
||||
return rule;
|
||||
}
|
||||
|
||||
private SingleAlert groupAlert(String fingerprint, String status) {
|
||||
return SingleAlert.builder()
|
||||
.fingerprint(fingerprint)
|
||||
.status(status)
|
||||
.labels(createLabels("severity", "critical"))
|
||||
.annotations(Map.of())
|
||||
.build();
|
||||
}
|
||||
|
||||
private Map<String, String> createLabels(String... keyValues) {
|
||||
Map<String, String> labels = new HashMap<>();
|
||||
for (int i = 0; i < keyValues.length; i += 2) {
|
||||
@@ -196,7 +348,7 @@ class AlarmGroupReduceTest {
|
||||
AtomicBoolean virtualThread, CountDownLatch firstStarted,
|
||||
CountDownLatch releaseFirst, CountDownLatch secondStarted,
|
||||
AtomicInteger maxConcurrent, AtomicInteger invocations) {
|
||||
super(alarmInhibitReduce, alertGroupConvergeDao, properties, false);
|
||||
super(alarmInhibitReduce, alertGroupConvergeDao, properties);
|
||||
this.virtualThreadLatch = virtualThreadLatch;
|
||||
this.virtualThread = virtualThread;
|
||||
this.firstStarted = firstStarted;
|
||||
|
||||
+65
-3
@@ -41,8 +41,12 @@ package org.apache.hertzbeat.alert.reduce;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.clearInvocations;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
@@ -73,6 +77,21 @@ import org.mockito.MockitoAnnotations;
|
||||
*/
|
||||
class AlarmInhibitReduceTest {
|
||||
|
||||
@Test
|
||||
void constructorIsPassiveAndLifecycleIsIdempotent() {
|
||||
clearInvocations(alertInhibitDao);
|
||||
AlarmInhibitReduce inactive = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
|
||||
new VirtualThreadProperties());
|
||||
|
||||
verifyNoInteractions(alertInhibitDao);
|
||||
|
||||
inactive.start();
|
||||
inactive.start();
|
||||
verify(alertInhibitDao, times(1)).findAlertInhibitsByEnableIsTrue();
|
||||
inactive.destroy();
|
||||
inactive.destroy();
|
||||
}
|
||||
|
||||
@Mock
|
||||
private AlertInhibitDao alertInhibitDao;
|
||||
|
||||
@@ -96,7 +115,8 @@ class AlarmInhibitReduceTest {
|
||||
when(alerterProperties.getInhibit()).thenReturn(inhibitProperties);
|
||||
|
||||
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
|
||||
new VirtualThreadProperties(), false);
|
||||
new VirtualThreadProperties());
|
||||
alarmInhibitReduce.start();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
@@ -214,6 +234,15 @@ class AlarmInhibitReduceTest {
|
||||
verify(alarmSilenceReduce).silenceAlarm(alert);
|
||||
}
|
||||
|
||||
@Test
|
||||
void synchronousSilenceOrStoreFailureIsReportedToGroupOwner() {
|
||||
GroupAlert alert = GroupAlert.builder().alerts(new ArrayList<>()).build();
|
||||
doThrow(new IllegalStateException("store unavailable"))
|
||||
.when(alarmSilenceReduce).silenceAlarm(alert);
|
||||
|
||||
assertFalse(alarmInhibitReduce.inhibitAlarm(alert));
|
||||
}
|
||||
|
||||
@Test
|
||||
void whenMultipleSourceAlerts_shouldInhibitAllMatchingTargets() {
|
||||
AlertInhibit rule = AlertInhibit.builder()
|
||||
@@ -306,7 +335,8 @@ class AlarmInhibitReduceTest {
|
||||
when(alerterProperties.getInhibit()).thenReturn(inhibitProperties);
|
||||
alarmInhibitReduce.destroy();
|
||||
alarmInhibitReduce = new AlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
|
||||
new VirtualThreadProperties(), false);
|
||||
new VirtualThreadProperties());
|
||||
alarmInhibitReduce.start();
|
||||
|
||||
AlertInhibit rule = AlertInhibit.builder()
|
||||
.id(1L)
|
||||
@@ -347,6 +377,7 @@ class AlarmInhibitReduceTest {
|
||||
alarmInhibitReduce.destroy();
|
||||
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
|
||||
new VirtualThreadProperties(), latch, virtualThread, null, null, null, null, null);
|
||||
alarmInhibitReduce.start();
|
||||
|
||||
alarmInhibitReduce.dispatchCleanupCache();
|
||||
|
||||
@@ -354,6 +385,13 @@ class AlarmInhibitReduceTest {
|
||||
assertTrue(virtualThread.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void dispatchAfterDestroyIsSafeNoOp() {
|
||||
alarmInhibitReduce.destroy();
|
||||
|
||||
alarmInhibitReduce.dispatchCleanupCache();
|
||||
}
|
||||
|
||||
@Test
|
||||
void dispatchCleanupCacheDoesNotRunConcurrently() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
@@ -364,6 +402,7 @@ class AlarmInhibitReduceTest {
|
||||
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
|
||||
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
|
||||
maxConcurrent, new AtomicInteger());
|
||||
alarmInhibitReduce.start();
|
||||
|
||||
alarmInhibitReduce.dispatchCleanupCache();
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
@@ -376,6 +415,29 @@ class AlarmInhibitReduceTest {
|
||||
assertEquals(1, maxConcurrent.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void destroyWhileCleanupIsRunningDropsPendingDispatch() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch secondStarted = new CountDownLatch(1);
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
alarmInhibitReduce.destroy();
|
||||
alarmInhibitReduce = new TestAlarmInhibitReduce(alarmSilenceReduce, alertInhibitDao, alerterProperties,
|
||||
new VirtualThreadProperties(), null, null, firstStarted, releaseFirst, secondStarted,
|
||||
new AtomicInteger(), invocations);
|
||||
alarmInhibitReduce.start();
|
||||
|
||||
alarmInhibitReduce.dispatchCleanupCache();
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
alarmInhibitReduce.dispatchCleanupCache();
|
||||
|
||||
alarmInhibitReduce.destroy();
|
||||
alarmInhibitReduce.dispatchCleanupCache();
|
||||
|
||||
assertFalse(secondStarted.await(500, TimeUnit.MILLISECONDS));
|
||||
assertEquals(1, invocations.get());
|
||||
}
|
||||
|
||||
private GroupAlert createGroupAlert(String status, Map<String, String> labels, List<SingleAlert> alerts) {
|
||||
return GroupAlert.builder()
|
||||
.status(status)
|
||||
@@ -424,7 +486,7 @@ class AlarmInhibitReduceTest {
|
||||
CountDownLatch firstStarted, CountDownLatch releaseFirst,
|
||||
CountDownLatch secondStarted, AtomicInteger maxConcurrent,
|
||||
AtomicInteger invocations) {
|
||||
super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties, false);
|
||||
super(alarmSilenceReduce, alertInhibitDao, alerterProperties, properties);
|
||||
this.virtualThreadLatch = virtualThreadLatch;
|
||||
this.virtualThread = virtualThread;
|
||||
this.firstStarted = firstStarted;
|
||||
|
||||
+17
-1
@@ -22,6 +22,8 @@ import org.apache.hertzbeat.collector.dispatch.CollectorRuntimeStatusProvider;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.FailureCode;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.ObservedLong;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayStatus;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayTransport;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.RuntimeTelemetry;
|
||||
|
||||
/**
|
||||
@@ -84,10 +86,24 @@ public class OtelRuntimeStatusProvider implements CollectorRuntimeStatusProvider
|
||||
diagnosticsReader.sanitize(snapshot.lastError(), properties),
|
||||
failureCode,
|
||||
telemetry,
|
||||
sources
|
||||
sources,
|
||||
otlpGateway(snapshot)
|
||||
);
|
||||
}
|
||||
|
||||
private OtlpGatewayStatus otlpGateway(OtelRuntimeSnapshot snapshot) {
|
||||
if (!properties.isOtlpGatewayEnabled()) {
|
||||
return OtlpGatewayStatus.disabled();
|
||||
}
|
||||
if (!properties.isEnabled() || snapshot.state() != OtelRuntimeState.RUNNING) {
|
||||
return OtlpGatewayStatus.unavailable();
|
||||
}
|
||||
// Gateway mode renders both OTLP receivers into the same health-checked runtime.
|
||||
return OtlpGatewayStatus.available(List.of(
|
||||
OtlpGatewayTransport.HTTP_PROTOBUF,
|
||||
OtlpGatewayTransport.GRPC));
|
||||
}
|
||||
|
||||
private List<ManagedOtelRuntimeStatus.ManagedOtelSourceStatus> sanitize(
|
||||
List<ManagedOtelRuntimeStatus.ManagedOtelSourceStatus> sources) {
|
||||
return sources.stream()
|
||||
|
||||
+36
@@ -41,6 +41,7 @@ class OtelRuntimeStatusProviderTest {
|
||||
properties.setEnabled(true);
|
||||
properties.setConfigRevision(12);
|
||||
properties.setToken("managed-intake-token");
|
||||
properties.setOtlpGatewayEnabled(true);
|
||||
OtelRuntimeSupervisor supervisor = mock(OtelRuntimeSupervisor.class);
|
||||
when(supervisor.snapshot()).thenReturn(new OtelRuntimeSnapshot(
|
||||
OtelRuntimeState.RUNNING, 42, 2, Instant.parse("2026-07-15T06:00:00Z"), ""));
|
||||
@@ -72,6 +73,12 @@ class OtelRuntimeStatusProviderTest {
|
||||
status.intakeCredentialState());
|
||||
assertEquals(List.of(source), status.sources());
|
||||
assertEquals(telemetry, status.telemetry());
|
||||
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.AVAILABLE,
|
||||
status.otlpGateway().state());
|
||||
assertEquals(List.of(
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF,
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC),
|
||||
status.otlpGateway().supportedTransports());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -93,6 +100,35 @@ class OtelRuntimeStatusProviderTest {
|
||||
|
||||
assertEquals(ManagedOtelRuntimeStatus.IntakeCredentialState.NOT_REQUIRED,
|
||||
status.intakeCredentialState());
|
||||
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.DISABLED,
|
||||
status.otlpGateway().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void enabledGatewayIsUnavailableUntilTheManagedRuntimeIsRunning() {
|
||||
OtelRuntimeProperties properties = new OtelRuntimeProperties();
|
||||
properties.setEnabled(true);
|
||||
properties.setOtlpGatewayEnabled(true);
|
||||
properties.setToken("managed-intake-token");
|
||||
OtelRuntimeSupervisor supervisor = mock(OtelRuntimeSupervisor.class);
|
||||
when(supervisor.snapshot()).thenReturn(new OtelRuntimeSnapshot(
|
||||
OtelRuntimeState.STARTING, -1, 0, Instant.parse("2026-07-15T06:00:00Z"), ""));
|
||||
when(supervisor.sourceStatuses()).thenReturn(List.of());
|
||||
OtelRuntimeDiagnosticsReader diagnosticsReader = mock(OtelRuntimeDiagnosticsReader.class);
|
||||
when(diagnosticsReader.latestFailure(properties)).thenReturn(ManagedOtelRuntimeStatus.FailureCode.NONE);
|
||||
when(diagnosticsReader.sanitize("", properties)).thenReturn("");
|
||||
OtelRuntimeStatusProvider provider = new OtelRuntimeStatusProvider(
|
||||
properties,
|
||||
supervisor,
|
||||
mock(OtelRuntimeTelemetryClient.class),
|
||||
diagnosticsReader,
|
||||
new OtelRuntimeFailureClassifier());
|
||||
|
||||
ManagedOtelRuntimeStatus status = provider.status();
|
||||
|
||||
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.UNAVAILABLE,
|
||||
status.otlpGateway().state());
|
||||
assertTrue(status.otlpGateway().supportedTransports().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.concurrent;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
/**
|
||||
* Process-local admission gate for work that must drain without owning its executor.
|
||||
*/
|
||||
public final class WorkAdmissionGate {
|
||||
|
||||
private final Object lock = new Object();
|
||||
private boolean accepting = true;
|
||||
private boolean stopped;
|
||||
private int activeWork;
|
||||
private int waitingWork;
|
||||
|
||||
/**
|
||||
* Returns a permit for admitted work, or {@code null} while admission is paused.
|
||||
*/
|
||||
public Permit tryAcquire() {
|
||||
synchronized (lock) {
|
||||
if (!accepting) {
|
||||
return null;
|
||||
}
|
||||
activeWork++;
|
||||
return new Permit(this);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserves replay work while ordinary admission is still paused.
|
||||
*/
|
||||
public Permit reserveReplay() {
|
||||
synchronized (lock) {
|
||||
if (stopped) {
|
||||
return null;
|
||||
}
|
||||
activeWork++;
|
||||
return new Permit(this);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for resumed admission, or returns {@code null} after terminal stop.
|
||||
*/
|
||||
public Permit awaitAcquire() throws InterruptedException {
|
||||
synchronized (lock) {
|
||||
while (!accepting && !stopped) {
|
||||
waitingWork++;
|
||||
try {
|
||||
lock.wait();
|
||||
} finally {
|
||||
waitingWork--;
|
||||
}
|
||||
}
|
||||
if (stopped) {
|
||||
return null;
|
||||
}
|
||||
activeWork++;
|
||||
return new Permit(this);
|
||||
}
|
||||
}
|
||||
|
||||
public void pauseAdmission() {
|
||||
synchronized (lock) {
|
||||
accepting = false;
|
||||
}
|
||||
}
|
||||
|
||||
public void awaitDrained(long timeoutNanos) throws InterruptedException, TimeoutException {
|
||||
synchronized (lock) {
|
||||
long remainingNanos = timeoutNanos;
|
||||
long startedNanos = System.nanoTime();
|
||||
while (activeWork > 0) {
|
||||
if (remainingNanos <= 0) {
|
||||
throw new TimeoutException();
|
||||
}
|
||||
TimeUnit.NANOSECONDS.timedWait(lock, remainingNanos);
|
||||
long elapsedNanos = System.nanoTime() - startedNanos;
|
||||
if (elapsedNanos <= 0) {
|
||||
remainingNanos = timeoutNanos;
|
||||
} else if (elapsedNanos >= timeoutNanos) {
|
||||
remainingNanos = 0;
|
||||
} else {
|
||||
remainingNanos = timeoutNanos - elapsedNanos;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void resumeAdmission() {
|
||||
synchronized (lock) {
|
||||
if (!stopped) {
|
||||
accepting = true;
|
||||
}
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Permanently rejects admission and wakes work waiting for a maintenance resume.
|
||||
*/
|
||||
public void stop() {
|
||||
synchronized (lock) {
|
||||
stopped = true;
|
||||
accepting = false;
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
int waitingWork() {
|
||||
synchronized (lock) {
|
||||
return waitingWork;
|
||||
}
|
||||
}
|
||||
|
||||
private void release() {
|
||||
synchronized (lock) {
|
||||
if (activeWork > 0) {
|
||||
activeWork--;
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent ownership token for one admitted unit of work.
|
||||
*/
|
||||
public static final class Permit implements AutoCloseable {
|
||||
|
||||
private final WorkAdmissionGate gate;
|
||||
private boolean closed;
|
||||
|
||||
private Permit(WorkAdmissionGate gate) {
|
||||
this.gate = gate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
if (!closed) {
|
||||
closed = true;
|
||||
gate.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-9
@@ -57,21 +57,18 @@ public interface CommonConstants {
|
||||
*/
|
||||
byte LOGIN_FAILED_CODE = 0x05;
|
||||
|
||||
/**
|
||||
* Monitoring status 0: Paused, 1: Up, 2: Down
|
||||
*/
|
||||
/** Monitoring status 0: Paused. */
|
||||
byte MONITOR_PAUSED_CODE = 0x00;
|
||||
|
||||
/**
|
||||
* Monitoring status 0: Paused, 1: Up, 2: Down
|
||||
*/
|
||||
/** Monitoring status 1: Up. */
|
||||
byte MONITOR_UP_CODE = 0x01;
|
||||
|
||||
/**
|
||||
* Monitoring status 0: Paused, 1: Up, 2: Down
|
||||
*/
|
||||
/** Monitoring status 2: Down. */
|
||||
byte MONITOR_DOWN_CODE = 0x02;
|
||||
|
||||
/** Monitoring status 3: Scheduled and waiting for the first availability result. */
|
||||
byte MONITOR_PENDING_CODE = 0x03;
|
||||
|
||||
/**
|
||||
* scrape type static
|
||||
*/
|
||||
|
||||
+76
-4
@@ -18,6 +18,7 @@
|
||||
package org.apache.hertzbeat.common.entity.dto;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -34,10 +35,12 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
|
||||
IntakeCredentialState intakeCredentialState,
|
||||
int restartCount, Instant changedAt, String lastError,
|
||||
FailureCode failureCode, RuntimeTelemetry telemetry,
|
||||
List<ManagedOtelSourceStatus> sources) {
|
||||
List<ManagedOtelSourceStatus> sources,
|
||||
OtlpGatewayStatus otlpGateway) {
|
||||
|
||||
public static final int CURRENT_SCHEMA_VERSION = 2;
|
||||
public static final int CURRENT_SCHEMA_VERSION = 3;
|
||||
private static final int LEGACY_SCHEMA_VERSION = 1;
|
||||
private static final int OTLP_GATEWAY_SCHEMA_VERSION = 3;
|
||||
private static final int MAXIMUM_DIAGNOSTIC_LENGTH = 512;
|
||||
// Active sources plus both sides of one pending/rejected replacement revision.
|
||||
private static final int MAXIMUM_SOURCE_STATUSES = 147;
|
||||
@@ -49,7 +52,7 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
|
||||
int restartCount, Instant changedAt, String lastError) {
|
||||
this(schemaVersion, enabled, state, desiredRevision, activeRevision, -1,
|
||||
intakeCredentialState, restartCount, changedAt, lastError, FailureCode.NONE,
|
||||
RuntimeTelemetry.unavailable(false), List.of());
|
||||
RuntimeTelemetry.unavailable(false), List.of(), OtlpGatewayStatus.notReported());
|
||||
}
|
||||
|
||||
public ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, RuntimeState state,
|
||||
@@ -59,7 +62,20 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
|
||||
List<ManagedOtelSourceStatus> sources) {
|
||||
this(schemaVersion, enabled, state, desiredRevision, activeRevision, -1,
|
||||
intakeCredentialState, restartCount, changedAt, lastError, FailureCode.NONE,
|
||||
RuntimeTelemetry.unavailable(hasFileSource(sources)), sources);
|
||||
RuntimeTelemetry.unavailable(hasFileSource(sources)), sources,
|
||||
OtlpGatewayStatus.notReported());
|
||||
}
|
||||
|
||||
/** Preserves the schema 1/2 constructor shape while gateway reporting rolls out. */
|
||||
public ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, RuntimeState state,
|
||||
long desiredRevision, long activeRevision, long pid,
|
||||
IntakeCredentialState intakeCredentialState,
|
||||
int restartCount, Instant changedAt, String lastError,
|
||||
FailureCode failureCode, RuntimeTelemetry telemetry,
|
||||
List<ManagedOtelSourceStatus> sources) {
|
||||
this(schemaVersion, enabled, state, desiredRevision, activeRevision, pid,
|
||||
intakeCredentialState, restartCount, changedAt, lastError, failureCode,
|
||||
telemetry, sources, OtlpGatewayStatus.notReported());
|
||||
}
|
||||
|
||||
public ManagedOtelRuntimeStatus {
|
||||
@@ -91,6 +107,11 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
|
||||
}
|
||||
failureCode = Objects.requireNonNullElse(failureCode, FailureCode.NONE);
|
||||
telemetry = telemetry == null ? RuntimeTelemetry.unavailable(hasFileSource(sources)) : telemetry;
|
||||
otlpGateway = otlpGateway == null ? OtlpGatewayStatus.notReported() : otlpGateway;
|
||||
if (schemaVersion < OTLP_GATEWAY_SCHEMA_VERSION
|
||||
&& otlpGateway.state() != OtlpGatewayState.NOT_REPORTED) {
|
||||
throw new IllegalArgumentException("OTLP Gateway status requires managed runtime status schema 3");
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean hasFileSource(List<ManagedOtelSourceStatus> sources) {
|
||||
@@ -134,6 +155,57 @@ public record ManagedOtelRuntimeStatus(int schemaVersion, boolean enabled, Runti
|
||||
UNKNOWN
|
||||
}
|
||||
|
||||
/** Lifecycle of the explicitly enabled application-telemetry Gateway listener. */
|
||||
public enum OtlpGatewayState {
|
||||
NOT_REPORTED,
|
||||
DISABLED,
|
||||
UNAVAILABLE,
|
||||
AVAILABLE
|
||||
}
|
||||
|
||||
/** Payload-free OTLP protocols confirmed by the running managed runtime. */
|
||||
public enum OtlpGatewayTransport {
|
||||
HTTP_PROTOBUF,
|
||||
GRPC
|
||||
}
|
||||
|
||||
/** Bounded runtime proof used to validate separately advertised public endpoints. */
|
||||
public record OtlpGatewayStatus(OtlpGatewayState state,
|
||||
List<OtlpGatewayTransport> supportedTransports) {
|
||||
|
||||
public OtlpGatewayStatus {
|
||||
state = Objects.requireNonNull(state, "state");
|
||||
supportedTransports = supportedTransports == null
|
||||
? List.of()
|
||||
: supportedTransports.stream()
|
||||
.distinct()
|
||||
.sorted(Comparator.comparingInt(OtlpGatewayTransport::ordinal))
|
||||
.toList();
|
||||
if (state == OtlpGatewayState.AVAILABLE && supportedTransports.isEmpty()) {
|
||||
throw new IllegalArgumentException("Available OTLP Gateway must report a transport");
|
||||
}
|
||||
if (state != OtlpGatewayState.AVAILABLE && !supportedTransports.isEmpty()) {
|
||||
throw new IllegalArgumentException("Unavailable OTLP Gateway cannot report transports");
|
||||
}
|
||||
}
|
||||
|
||||
public static OtlpGatewayStatus notReported() {
|
||||
return new OtlpGatewayStatus(OtlpGatewayState.NOT_REPORTED, List.of());
|
||||
}
|
||||
|
||||
public static OtlpGatewayStatus disabled() {
|
||||
return new OtlpGatewayStatus(OtlpGatewayState.DISABLED, List.of());
|
||||
}
|
||||
|
||||
public static OtlpGatewayStatus unavailable() {
|
||||
return new OtlpGatewayStatus(OtlpGatewayState.UNAVAILABLE, List.of());
|
||||
}
|
||||
|
||||
public static OtlpGatewayStatus available(List<OtlpGatewayTransport> transports) {
|
||||
return new OtlpGatewayStatus(OtlpGatewayState.AVAILABLE, transports);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Availability of one numeric runtime metric. Zero is meaningful only when available.
|
||||
*/
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.runtime;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
/** Represents the business-runtime boundary decided before an application context starts. */
|
||||
public final class BusinessRuntimeGate {
|
||||
|
||||
private final RuntimeMode mode;
|
||||
|
||||
private BusinessRuntimeGate(RuntimeMode mode) {
|
||||
this.mode = Objects.requireNonNull(mode, "mode");
|
||||
}
|
||||
|
||||
public static BusinessRuntimeGate fixed(RuntimeMode mode) {
|
||||
return new BusinessRuntimeGate(mode);
|
||||
}
|
||||
|
||||
public RuntimeMode mode() {
|
||||
return mode;
|
||||
}
|
||||
|
||||
public boolean isOpen() {
|
||||
return mode == RuntimeMode.NORMAL;
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.runtime;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/** Application runtime scope selected before a Spring context starts. */
|
||||
public enum RuntimeMode {
|
||||
SETUP_ONLY("setup_only"),
|
||||
FULL_SETUP_GATED("full_setup_gated"),
|
||||
NORMAL("normal"),
|
||||
RECOVERY("recovery");
|
||||
|
||||
public static final String PROPERTY_NAME = "hertzbeat.runtime.mode";
|
||||
|
||||
private final String value;
|
||||
|
||||
RuntimeMode(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public String value() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public static RuntimeMode fromProperty(String value) {
|
||||
if (value == null) {
|
||||
return NORMAL;
|
||||
}
|
||||
String normalized = value.trim().toLowerCase(Locale.ROOT);
|
||||
for (RuntimeMode mode : values()) {
|
||||
if (mode.value.equals(normalized)) {
|
||||
return mode;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unsupported runtime mode");
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.concurrent;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class WorkAdmissionGateTest {
|
||||
|
||||
@Test
|
||||
void pauseRejectsNewWorkAndDrainsOnlyAdmittedWork() throws Exception {
|
||||
WorkAdmissionGate gate = new WorkAdmissionGate();
|
||||
WorkAdmissionGate.Permit admitted = gate.tryAcquire();
|
||||
|
||||
gate.pauseAdmission();
|
||||
|
||||
assertThat(gate.tryAcquire()).isNull();
|
||||
assertThatThrownBy(() -> gate.awaitDrained(0))
|
||||
.isInstanceOf(TimeoutException.class);
|
||||
|
||||
admitted.close();
|
||||
gate.awaitDrained(TimeUnit.SECONDS.toNanos(1));
|
||||
gate.resumeAdmission();
|
||||
|
||||
assertThat(gate.tryAcquire()).isNotNull().satisfies(WorkAdmissionGate.Permit::close);
|
||||
}
|
||||
|
||||
@Test
|
||||
void repeatedPauseResumeDoesNotDuplicatePermits() throws Exception {
|
||||
WorkAdmissionGate gate = new WorkAdmissionGate();
|
||||
|
||||
gate.pauseAdmission();
|
||||
gate.pauseAdmission();
|
||||
gate.awaitDrained(0);
|
||||
gate.resumeAdmission();
|
||||
gate.resumeAdmission();
|
||||
|
||||
WorkAdmissionGate.Permit permit = gate.tryAcquire();
|
||||
assertThat(permit).isNotNull();
|
||||
permit.close();
|
||||
gate.awaitDrained(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void waitingAdmissionResumesOnceAndInterruptDoesNotLeakPermit() throws Exception {
|
||||
WorkAdmissionGate gate = new WorkAdmissionGate();
|
||||
gate.pauseAdmission();
|
||||
AtomicReference<WorkAdmissionGate.Permit> resumed = new AtomicReference<>();
|
||||
Thread waiter = Thread.ofPlatform().start(() -> {
|
||||
try {
|
||||
resumed.set(gate.awaitAcquire());
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
while (gate.waitingWork() == 0) {
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
|
||||
gate.resumeAdmission();
|
||||
waiter.join(1_000);
|
||||
assertThat(waiter.isAlive()).isFalse();
|
||||
assertThat(resumed.get()).isNotNull();
|
||||
resumed.get().close();
|
||||
gate.awaitDrained(0);
|
||||
|
||||
gate.pauseAdmission();
|
||||
AtomicBoolean interrupted = new AtomicBoolean();
|
||||
Thread interruptedWaiter = Thread.ofPlatform().start(() -> {
|
||||
try {
|
||||
gate.awaitAcquire();
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
interrupted.set(Thread.currentThread().isInterrupted());
|
||||
}
|
||||
});
|
||||
while (gate.waitingWork() == 0) {
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
interruptedWaiter.interrupt();
|
||||
interruptedWaiter.join(1_000);
|
||||
|
||||
assertThat(interrupted.get()).isTrue();
|
||||
gate.awaitDrained(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalStopWakesWaiterAndResumeCannotReviveAdmission() throws Exception {
|
||||
WorkAdmissionGate gate = new WorkAdmissionGate();
|
||||
gate.pauseAdmission();
|
||||
AtomicReference<WorkAdmissionGate.Permit> result = new AtomicReference<>();
|
||||
Thread waiter = Thread.ofPlatform().start(() -> {
|
||||
try {
|
||||
result.set(gate.awaitAcquire());
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
while (gate.waitingWork() == 0) {
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
|
||||
gate.stop();
|
||||
gate.resumeAdmission();
|
||||
waiter.join(1_000);
|
||||
|
||||
assertThat(waiter.isAlive()).isFalse();
|
||||
assertThat(result.get()).isNull();
|
||||
assertThat(gate.tryAcquire()).isNull();
|
||||
}
|
||||
}
|
||||
+33
@@ -190,5 +190,38 @@ class ManagedOtelRuntimeStatusTest {
|
||||
status.telemetry().queueSize().state());
|
||||
assertEquals(ManagedOtelRuntimeStatus.ValueState.UNAVAILABLE,
|
||||
status.telemetry().queueCapacityBySignal().traces().state());
|
||||
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.NOT_REPORTED,
|
||||
status.otlpGateway().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void carriesOnlyBoundedPayloadFreeOtlpGatewayCapabilities() {
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayStatus gateway =
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayStatus.available(List.of(
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF,
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC));
|
||||
ManagedOtelRuntimeStatus status = new ManagedOtelRuntimeStatus(
|
||||
ManagedOtelRuntimeStatus.CURRENT_SCHEMA_VERSION,
|
||||
true,
|
||||
ManagedOtelRuntimeStatus.RuntimeState.RUNNING,
|
||||
1,
|
||||
1,
|
||||
-1,
|
||||
ManagedOtelRuntimeStatus.IntakeCredentialState.CONFIGURED,
|
||||
0,
|
||||
Instant.now(),
|
||||
"",
|
||||
ManagedOtelRuntimeStatus.FailureCode.NONE,
|
||||
ManagedOtelRuntimeStatus.RuntimeTelemetry.unavailable(false),
|
||||
List.of(),
|
||||
gateway);
|
||||
|
||||
assertEquals(ManagedOtelRuntimeStatus.OtlpGatewayState.AVAILABLE,
|
||||
status.otlpGateway().state());
|
||||
assertEquals(List.of(
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayTransport.HTTP_PROTOBUF,
|
||||
ManagedOtelRuntimeStatus.OtlpGatewayTransport.GRPC),
|
||||
status.otlpGateway().supportedTransports());
|
||||
assertFalse(status.toString().contains("endpoint"));
|
||||
}
|
||||
}
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.runtime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class BusinessRuntimeGateTest {
|
||||
|
||||
@Test
|
||||
void opensOnlyForNormalRuntime() {
|
||||
for (RuntimeMode mode : RuntimeMode.values()) {
|
||||
BusinessRuntimeGate gate = BusinessRuntimeGate.fixed(mode);
|
||||
assertEquals(mode, gate.mode());
|
||||
assertEquals(mode == RuntimeMode.NORMAL, gate.isOpen());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingModePreservesExistingNormalStartup() {
|
||||
assertEquals(RuntimeMode.NORMAL, RuntimeMode.fromProperty(null));
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,11 @@
|
||||
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
||||
+4
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.alerter;
|
||||
|
||||
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Convert;
|
||||
@@ -72,16 +73,19 @@ public class GroupAlert {
|
||||
@Schema(title = "Group Labels", example = "{\"alertname\": \"HighCPUUsage\"}")
|
||||
@Convert(converter = JsonMapAttributeConverter.class)
|
||||
@Column(length = 2048)
|
||||
@JsonInclude(content = JsonInclude.Include.NON_NULL)
|
||||
private Map<String, String> groupLabels;
|
||||
|
||||
@Schema(title = "Common Labels", example = "{\"alertname\": \"HighCPUUsage\", \"instance\": \"server1\", \"severity\": \"critical\"}")
|
||||
@Convert(converter = JsonMapAttributeConverter.class)
|
||||
@Column(length = 2048)
|
||||
@JsonInclude(content = JsonInclude.Include.NON_NULL)
|
||||
private Map<String, String> commonLabels;
|
||||
|
||||
@Schema(title = "Common Annotations", example = "{\"summary\": \"High CPU usage detected\", \"description\": \"CPU usage is back to normal for server1\"}")
|
||||
@Convert(converter = JsonMapAttributeConverter.class)
|
||||
@Column(columnDefinition = "TEXT")
|
||||
@JsonInclude(content = JsonInclude.Include.NON_NULL)
|
||||
private Map<String, String> commonAnnotations;
|
||||
|
||||
@Schema(title = "Alert Fingerprints", example = "[\"dxsdfdsf\"]")
|
||||
|
||||
+3
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.common.entity.alerter;
|
||||
|
||||
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
|
||||
import com.fasterxml.jackson.annotation.JsonFormat;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
import jakarta.persistence.Column;
|
||||
import jakarta.persistence.Convert;
|
||||
@@ -67,11 +68,13 @@ public class SingleAlert {
|
||||
@Schema(title = "Labels", example = "{\"alertname\": \"HighCPUUsage\", \"priority\": \"critical\", \"instance\": \"343483943\"}")
|
||||
@Convert(converter = JsonMapAttributeConverter.class)
|
||||
@Column(length = 2048)
|
||||
@JsonInclude(content = JsonInclude.Include.NON_NULL)
|
||||
private Map<String, String> labels;
|
||||
|
||||
@Schema(title = "Annotations", example = "{\"summary\": \"High CPU usage detected\"}")
|
||||
@Convert(converter = JsonMapAttributeConverter.class)
|
||||
@Column(length = 4096)
|
||||
@JsonInclude(content = JsonInclude.Include.NON_NULL)
|
||||
private Map<String, String> annotations;
|
||||
|
||||
@Schema(title = "Content", example = "CPU usage is above 80% for the last 5 minutes on instance server1.example.com.")
|
||||
|
||||
+1
-1
@@ -98,7 +98,7 @@ public class Monitor {
|
||||
@Size(max = 100)
|
||||
private String cronExpression;
|
||||
|
||||
@Schema(title = "Task status 0: Paused, 1: Up, 2: Down", accessMode = READ_WRITE)
|
||||
@Schema(title = "Task status 0: Paused, 1: Up, 2: Down, 3: Pending", accessMode = READ_WRITE)
|
||||
@Min(0)
|
||||
@Max(4)
|
||||
private byte status;
|
||||
|
||||
+3
-1
@@ -25,6 +25,7 @@ import java.util.List;
|
||||
public interface ObservabilityAccessTokenGateway {
|
||||
|
||||
String CLAIM_MANAGED = "managed";
|
||||
String CLAIM_CREDENTIAL_VERSION = "credentialVersion";
|
||||
|
||||
/**
|
||||
* Check the status of a managed token.
|
||||
@@ -62,9 +63,10 @@ public interface ObservabilityAccessTokenGateway {
|
||||
*
|
||||
* @param userId subject from token
|
||||
* @param claimedRoles roles embedded in token
|
||||
* @param credentialVersion account credential generation embedded in token, or null for legacy accounts
|
||||
* @return null when owner is still allowed, otherwise rejection reason
|
||||
*/
|
||||
String checkManagedTokenAccess(String userId, List<String> claimedRoles);
|
||||
String checkManagedTokenAccess(String userId, List<String> claimedRoles, Long credentialVersion);
|
||||
|
||||
/**
|
||||
* Touch token last used time.
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.runtime;
|
||||
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.core.env.Environment;
|
||||
|
||||
/** Provides the immutable runtime gate selected before this context was launched. */
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
public class BusinessRuntimeConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public BusinessRuntimeGate businessRuntimeGate(Environment environment) {
|
||||
return BusinessRuntimeGate.fixed(RuntimeMode.fromProperty(environment.getProperty(RuntimeMode.PROPERTY_NAME)));
|
||||
}
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.runtime;
|
||||
|
||||
import java.lang.annotation.ElementType;
|
||||
import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
|
||||
/** Registers an active side-effect boundary only in the normal full runtime. */
|
||||
@Target({ElementType.TYPE, ElementType.METHOD})
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@ConditionalOnProperty(name = RuntimeMode.PROPERTY_NAME, havingValue = "normal", matchIfMissing = true)
|
||||
public @interface ConditionalOnNormalBusinessRuntime {
|
||||
}
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import org.aopalliance.intercept.MethodInterceptor;
|
||||
import org.springframework.aop.Pointcut;
|
||||
import org.springframework.aop.support.AbstractPointcutAdvisor;
|
||||
import org.springframework.aop.support.AopUtils;
|
||||
import org.springframework.aop.support.StaticMethodMatcherPointcut;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.data.repository.Repository;
|
||||
import org.springframework.transaction.TransactionDefinition;
|
||||
import org.springframework.transaction.interceptor.TransactionAttribute;
|
||||
import org.springframework.transaction.interceptor.TransactionAttributeSource;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
/** Admits writable transactional boundaries before Spring opens their transactions. */
|
||||
public final class MetadataWriteAdmissionAdvisor extends AbstractPointcutAdvisor {
|
||||
|
||||
private static final int ADVISOR_ORDER = Ordered.HIGHEST_PRECEDENCE + 100;
|
||||
|
||||
private final TransactionAttributeSource transactionAttributes;
|
||||
private final MetadataWriteAdmissionCoordinator coordinator;
|
||||
private final TransactionCompletionPermitRegistry transactionPermits;
|
||||
private final boolean repositoryAttributes;
|
||||
private final Pointcut pointcut = new TransactionAttributePointcut();
|
||||
private final MethodInterceptor advice = this::invoke;
|
||||
|
||||
MetadataWriteAdmissionAdvisor(
|
||||
TransactionAttributeSource transactionAttributes,
|
||||
MetadataWriteAdmissionCoordinator coordinator,
|
||||
TransactionCompletionPermitRegistry transactionPermits,
|
||||
boolean repositoryAttributes) {
|
||||
this.transactionAttributes = transactionAttributes;
|
||||
this.coordinator = coordinator;
|
||||
this.transactionPermits = transactionPermits;
|
||||
this.repositoryAttributes = repositoryAttributes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Pointcut getPointcut() {
|
||||
return pointcut;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MethodInterceptor getAdvice() {
|
||||
return advice;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return ADVISOR_ORDER;
|
||||
}
|
||||
|
||||
private Object invoke(org.aopalliance.intercept.MethodInvocation invocation) throws Throwable {
|
||||
TransactionAttribute attribute = resolveAttribute(invocation.getMethod(), invocation.getThis());
|
||||
if (attribute == null || attribute.isReadOnly()) {
|
||||
return invocation.proceed();
|
||||
}
|
||||
if (transactionPermits.hasPermit()) {
|
||||
return invocation.proceed();
|
||||
}
|
||||
if (joinsExistingPhysicalTransaction(attribute)) {
|
||||
MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction();
|
||||
transactionPermits.bind(permit);
|
||||
transactionPermits.beginInvocation();
|
||||
try {
|
||||
return invocation.proceed();
|
||||
} finally {
|
||||
transactionPermits.endInvocation();
|
||||
}
|
||||
}
|
||||
try (MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction()) {
|
||||
transactionPermits.beginInvocation();
|
||||
try {
|
||||
return invocation.proceed();
|
||||
} finally {
|
||||
transactionPermits.endInvocation();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private TransactionAttribute resolveAttribute(Method method, Object target) {
|
||||
Class<?> targetClass = target == null ? method.getDeclaringClass() : AopUtils.getTargetClass(target);
|
||||
return transactionAttributes.getTransactionAttribute(method, targetClass);
|
||||
}
|
||||
|
||||
private boolean joinsExistingPhysicalTransaction(TransactionAttribute attribute) {
|
||||
if (!TransactionSynchronizationManager.isActualTransactionActive()) {
|
||||
return false;
|
||||
}
|
||||
return switch (attribute.getPropagationBehavior()) {
|
||||
case TransactionDefinition.PROPAGATION_REQUIRED,
|
||||
TransactionDefinition.PROPAGATION_SUPPORTS,
|
||||
TransactionDefinition.PROPAGATION_MANDATORY,
|
||||
TransactionDefinition.PROPAGATION_NESTED -> true;
|
||||
default -> false;
|
||||
};
|
||||
}
|
||||
|
||||
private final class TransactionAttributePointcut extends StaticMethodMatcherPointcut {
|
||||
|
||||
@Override
|
||||
public boolean matches(Method method, Class<?> targetClass) {
|
||||
return (repositoryAttributes || !Repository.class.isAssignableFrom(targetClass))
|
||||
&& transactionAttributes.getTransactionAttribute(method, targetClass) != null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import org.springframework.beans.factory.config.BeanDefinition;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Role;
|
||||
import org.springframework.transaction.interceptor.TransactionAttributeSource;
|
||||
|
||||
/** Spring wiring for process-local metadata write admission. */
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
|
||||
public class MetadataWriteAdmissionConfiguration {
|
||||
|
||||
@Bean
|
||||
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
|
||||
MetadataWriteAdmissionCoordinator metadataWriteAdmissionCoordinator() {
|
||||
return new MetadataWriteAdmissionCoordinator();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
|
||||
TransactionCompletionPermitRegistry transactionCompletionPermitRegistry() {
|
||||
return new TransactionCompletionPermitRegistry();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
|
||||
MetadataWriteAdmissionAdvisor metadataWriteAdmissionAdvisor(
|
||||
TransactionAttributeSource transactionAttributeSource,
|
||||
MetadataWriteAdmissionCoordinator coordinator,
|
||||
TransactionCompletionPermitRegistry transactionPermits) {
|
||||
return new MetadataWriteAdmissionAdvisor(
|
||||
transactionAttributeSource, coordinator, transactionPermits, false);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
|
||||
static SpringDataWriteAdmissionBeanPostProcessor springDataWriteAdmissionBeanPostProcessor(
|
||||
MetadataWriteAdmissionCoordinator coordinator,
|
||||
TransactionCompletionPermitRegistry transactionPermits) {
|
||||
return new SpringDataWriteAdmissionBeanPostProcessor(coordinator, transactionPermits);
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
/** Coordinates process-local writable transaction admission and maintenance drain. */
|
||||
public final class MetadataWriteAdmissionCoordinator {
|
||||
|
||||
private final ReentrantLock lock = new ReentrantLock();
|
||||
private final Condition noActiveWrites = lock.newCondition();
|
||||
private MetadataWriteAdmissionPhase phase = MetadataWriteAdmissionPhase.OPEN;
|
||||
private String operationId;
|
||||
private long epoch;
|
||||
private Object leaseToken;
|
||||
private int activeWrites;
|
||||
|
||||
/** Drain admitted writes and enter maintenance for one operation. */
|
||||
public MetadataWriteMaintenanceLease acquire(String requestedOperationId, Duration timeout) {
|
||||
requireValid(requestedOperationId, timeout);
|
||||
long timeoutNanos = toNanos(timeout);
|
||||
lock.lock();
|
||||
try {
|
||||
if (phase != MetadataWriteAdmissionPhase.OPEN) {
|
||||
throw MetadataWriteAdmissionException.operationConflict();
|
||||
}
|
||||
phase = MetadataWriteAdmissionPhase.DRAINING;
|
||||
operationId = requestedOperationId;
|
||||
long currentEpoch = ++epoch;
|
||||
Object currentToken = new Object();
|
||||
leaseToken = currentToken;
|
||||
long remainingNanos = timeoutNanos;
|
||||
while (activeWrites > 0) {
|
||||
if (remainingNanos <= 0) {
|
||||
reopen(currentEpoch, currentToken);
|
||||
throw MetadataWriteAdmissionException.drainTimeout();
|
||||
}
|
||||
try {
|
||||
remainingNanos = noActiveWrites.awaitNanos(remainingNanos);
|
||||
} catch (InterruptedException exception) {
|
||||
reopen(currentEpoch, currentToken);
|
||||
Thread.currentThread().interrupt();
|
||||
throw MetadataWriteAdmissionException.acquisitionInterrupted();
|
||||
}
|
||||
}
|
||||
phase = MetadataWriteAdmissionPhase.ACTIVE;
|
||||
return new MetadataWriteMaintenanceLease(this, requestedOperationId, currentEpoch, currentToken);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/** Return a consistent view without exposing the lease capability. */
|
||||
public MetadataWriteAdmissionSnapshot snapshot() {
|
||||
lock.lock();
|
||||
try {
|
||||
return new MetadataWriteAdmissionSnapshot(phase, operationId, epoch, activeWrites);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
TransactionPermit admitWritableTransaction() {
|
||||
lock.lock();
|
||||
try {
|
||||
if (phase != MetadataWriteAdmissionPhase.OPEN) {
|
||||
throw MetadataWriteAdmissionException.metadataWritesPaused();
|
||||
}
|
||||
activeWrites++;
|
||||
return new TransactionPermit(this);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void release(String releasedOperationId, long releasedEpoch, Object releasedToken) {
|
||||
lock.lock();
|
||||
try {
|
||||
if (phase == MetadataWriteAdmissionPhase.ACTIVE
|
||||
&& epoch == releasedEpoch
|
||||
&& operationId.equals(releasedOperationId)
|
||||
&& leaseToken == releasedToken) {
|
||||
phase = MetadataWriteAdmissionPhase.OPEN;
|
||||
operationId = null;
|
||||
leaseToken = null;
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void releaseWritableTransaction() {
|
||||
lock.lock();
|
||||
try {
|
||||
activeWrites--;
|
||||
if (activeWrites == 0) {
|
||||
noActiveWrites.signalAll();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void reopen(long failedEpoch, Object failedToken) {
|
||||
if (epoch == failedEpoch && leaseToken == failedToken && phase == MetadataWriteAdmissionPhase.DRAINING) {
|
||||
phase = MetadataWriteAdmissionPhase.OPEN;
|
||||
operationId = null;
|
||||
leaseToken = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void requireValid(String requestedOperationId, Duration timeout) {
|
||||
if (requestedOperationId == null || requestedOperationId.isBlank()
|
||||
|| timeout == null || timeout.isNegative()) {
|
||||
throw MetadataWriteAdmissionException.invalidRequest();
|
||||
}
|
||||
}
|
||||
|
||||
private long toNanos(Duration timeout) {
|
||||
try {
|
||||
return timeout.toNanos();
|
||||
} catch (ArithmeticException exception) {
|
||||
throw MetadataWriteAdmissionException.invalidRequest();
|
||||
}
|
||||
}
|
||||
|
||||
static final class TransactionPermit implements AutoCloseable {
|
||||
|
||||
private final MetadataWriteAdmissionCoordinator coordinator;
|
||||
private boolean closed;
|
||||
|
||||
private TransactionPermit(MetadataWriteAdmissionCoordinator coordinator) {
|
||||
this.coordinator = coordinator;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed) {
|
||||
closed = true;
|
||||
coordinator.releaseWritableTransaction();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
/** Stable, secret-free failure classifications for metadata write admission. */
|
||||
public enum MetadataWriteAdmissionErrorCode {
|
||||
MAINTENANCE_ACTIVE("metadata_writes_paused"),
|
||||
OPERATION_CONFLICT("operation_conflict"),
|
||||
DRAIN_TIMEOUT("drain_timeout"),
|
||||
ACQUISITION_INTERRUPTED("acquisition_interrupted"),
|
||||
INVALID_REQUEST("invalid_request");
|
||||
|
||||
private final String wireCode;
|
||||
|
||||
MetadataWriteAdmissionErrorCode(String wireCode) {
|
||||
this.wireCode = wireCode;
|
||||
}
|
||||
|
||||
/** Return the stable, safe code exposed at typed transport boundaries. */
|
||||
public String wireCode() {
|
||||
return wireCode;
|
||||
}
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
/** Safe admission failure that never exposes operation identifiers or persistence details. */
|
||||
public final class MetadataWriteAdmissionException extends RuntimeException {
|
||||
|
||||
private static final String MAINTENANCE_MESSAGE = "Metadata writes are temporarily unavailable";
|
||||
private static final String CONFLICT_MESSAGE = "Metadata maintenance operation is already active";
|
||||
private static final String TIMEOUT_MESSAGE = "Metadata write drain timed out";
|
||||
private static final String INTERRUPTED_MESSAGE = "Metadata write drain was interrupted";
|
||||
private static final String INVALID_MESSAGE = "Metadata maintenance request is invalid";
|
||||
|
||||
private final MetadataWriteAdmissionErrorCode code;
|
||||
|
||||
private MetadataWriteAdmissionException(MetadataWriteAdmissionErrorCode code, String message) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
/** Return the stable machine-readable classification. */
|
||||
public MetadataWriteAdmissionErrorCode code() {
|
||||
return code;
|
||||
}
|
||||
|
||||
/** Return the stable, secret-free message suitable for typed transport boundaries. */
|
||||
public String safeMessage() {
|
||||
return getMessage();
|
||||
}
|
||||
|
||||
/** Create the stable rejection used by typed metadata-write callers and tests. */
|
||||
public static MetadataWriteAdmissionException metadataWritesPaused() {
|
||||
return new MetadataWriteAdmissionException(
|
||||
MetadataWriteAdmissionErrorCode.MAINTENANCE_ACTIVE, MAINTENANCE_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataWriteAdmissionException operationConflict() {
|
||||
return new MetadataWriteAdmissionException(
|
||||
MetadataWriteAdmissionErrorCode.OPERATION_CONFLICT, CONFLICT_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataWriteAdmissionException drainTimeout() {
|
||||
return new MetadataWriteAdmissionException(
|
||||
MetadataWriteAdmissionErrorCode.DRAIN_TIMEOUT, TIMEOUT_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataWriteAdmissionException acquisitionInterrupted() {
|
||||
return new MetadataWriteAdmissionException(
|
||||
MetadataWriteAdmissionErrorCode.ACQUISITION_INTERRUPTED, INTERRUPTED_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataWriteAdmissionException invalidRequest() {
|
||||
return new MetadataWriteAdmissionException(
|
||||
MetadataWriteAdmissionErrorCode.INVALID_REQUEST, INVALID_MESSAGE);
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
/** Metadata write admission lifecycle. */
|
||||
public enum MetadataWriteAdmissionPhase {
|
||||
OPEN,
|
||||
DRAINING,
|
||||
ACTIVE
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
/** Immutable diagnostic projection of local metadata write admission state. */
|
||||
public record MetadataWriteAdmissionSnapshot(
|
||||
MetadataWriteAdmissionPhase phase,
|
||||
String operationId,
|
||||
long epoch,
|
||||
int activeWritableTransactions) {
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/** Capability that returns metadata write admission to OPEN when its matching epoch is released. */
|
||||
public final class MetadataWriteMaintenanceLease implements AutoCloseable {
|
||||
|
||||
private final MetadataWriteAdmissionCoordinator coordinator;
|
||||
private final String operationId;
|
||||
private final long epoch;
|
||||
private final Object token;
|
||||
private final AtomicBoolean closed = new AtomicBoolean();
|
||||
|
||||
MetadataWriteMaintenanceLease(
|
||||
MetadataWriteAdmissionCoordinator coordinator, String operationId, long epoch, Object token) {
|
||||
this.coordinator = coordinator;
|
||||
this.operationId = operationId;
|
||||
this.epoch = epoch;
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (closed.compareAndSet(false, true)) {
|
||||
coordinator.release(operationId, epoch, token);
|
||||
}
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import org.springframework.aop.Advisor;
|
||||
import org.springframework.aop.framework.Advised;
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.beans.factory.BeanInitializationException;
|
||||
import org.springframework.beans.factory.config.BeanPostProcessor;
|
||||
import org.springframework.core.Ordered;
|
||||
import org.springframework.data.repository.Repository;
|
||||
import org.springframework.transaction.interceptor.TransactionAttributeSource;
|
||||
import org.springframework.transaction.interceptor.TransactionInterceptor;
|
||||
import org.springframework.transaction.interceptor.TransactionalProxy;
|
||||
|
||||
/** Inserts write admission into each existing Spring Data transaction proxy. */
|
||||
final class SpringDataWriteAdmissionBeanPostProcessor implements BeanPostProcessor, Ordered {
|
||||
|
||||
private final MetadataWriteAdmissionCoordinator coordinator;
|
||||
private final TransactionCompletionPermitRegistry transactionPermits;
|
||||
|
||||
SpringDataWriteAdmissionBeanPostProcessor(
|
||||
MetadataWriteAdmissionCoordinator coordinator,
|
||||
TransactionCompletionPermitRegistry transactionPermits) {
|
||||
this.coordinator = coordinator;
|
||||
this.transactionPermits = transactionPermits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getOrder() {
|
||||
return Ordered.LOWEST_PRECEDENCE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
|
||||
if (!(bean instanceof Repository<?, ?>)
|
||||
|| !(bean instanceof TransactionalProxy)
|
||||
|| !(bean instanceof Advised advised)) {
|
||||
return bean;
|
||||
}
|
||||
if (hasAdmissionAdvisor(advised)) {
|
||||
return bean;
|
||||
}
|
||||
if (advised.isFrozen()) {
|
||||
throw new BeanInitializationException("Spring Data transaction proxy is frozen");
|
||||
}
|
||||
int transactionAdvisorIndex = transactionAdvisorIndex(advised);
|
||||
TransactionInterceptor interceptor = (TransactionInterceptor) advised
|
||||
.getAdvisors()[transactionAdvisorIndex].getAdvice();
|
||||
TransactionAttributeSource attributes = interceptor.getTransactionAttributeSource();
|
||||
if (attributes == null) {
|
||||
throw new BeanInitializationException("Spring Data transaction attributes are unavailable");
|
||||
}
|
||||
advised.addAdvisor(transactionAdvisorIndex,
|
||||
new MetadataWriteAdmissionAdvisor(attributes, coordinator, transactionPermits, true));
|
||||
return bean;
|
||||
}
|
||||
|
||||
private boolean hasAdmissionAdvisor(Advised advised) {
|
||||
for (Advisor advisor : advised.getAdvisors()) {
|
||||
if (advisor instanceof MetadataWriteAdmissionAdvisor) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private int transactionAdvisorIndex(Advised advised) {
|
||||
int found = -1;
|
||||
for (int index = 0; index < advised.getAdvisors().length; index++) {
|
||||
if (advised.getAdvisors()[index].getAdvice() instanceof TransactionInterceptor) {
|
||||
if (found >= 0) {
|
||||
throw new BeanInitializationException("Spring Data transaction advisor is ambiguous");
|
||||
}
|
||||
found = index;
|
||||
}
|
||||
}
|
||||
if (found < 0) {
|
||||
throw new BeanInitializationException("Spring Data transaction advisor is unavailable");
|
||||
}
|
||||
return found;
|
||||
}
|
||||
}
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
/** Binds a writable admission permit to the current physical transaction completion. */
|
||||
final class TransactionCompletionPermitRegistry {
|
||||
|
||||
private final Object resourceKey = new Object();
|
||||
private final SynchronizationRegistrar registrar;
|
||||
private final ThreadLocal<Boolean> invocationPermit = new ThreadLocal<>();
|
||||
|
||||
TransactionCompletionPermitRegistry() {
|
||||
this(TransactionSynchronizationManager::registerSynchronization);
|
||||
}
|
||||
|
||||
TransactionCompletionPermitRegistry(SynchronizationRegistrar registrar) {
|
||||
this.registrar = registrar;
|
||||
}
|
||||
|
||||
boolean hasPermit() {
|
||||
return invocationPermit.get() != null || TransactionSynchronizationManager.hasResource(resourceKey);
|
||||
}
|
||||
|
||||
void beginInvocation() {
|
||||
invocationPermit.set(Boolean.TRUE);
|
||||
}
|
||||
|
||||
void endInvocation() {
|
||||
invocationPermit.remove();
|
||||
}
|
||||
|
||||
void bind(MetadataWriteAdmissionCoordinator.TransactionPermit permit) {
|
||||
if (!TransactionSynchronizationManager.isActualTransactionActive()
|
||||
|| !TransactionSynchronizationManager.isSynchronizationActive()) {
|
||||
permit.close();
|
||||
throw new IllegalStateException("Transaction synchronization is unavailable");
|
||||
}
|
||||
boolean bound = false;
|
||||
try {
|
||||
TransactionSynchronizationManager.bindResource(resourceKey, permit);
|
||||
bound = true;
|
||||
registrar.register(new PermitReleaseSynchronization(permit));
|
||||
} catch (RuntimeException | Error failure) {
|
||||
if (bound) {
|
||||
TransactionSynchronizationManager.unbindResourceIfPossible(resourceKey);
|
||||
}
|
||||
permit.close();
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
interface SynchronizationRegistrar {
|
||||
|
||||
void register(TransactionSynchronization synchronization);
|
||||
}
|
||||
|
||||
private final class PermitReleaseSynchronization implements TransactionSynchronization {
|
||||
|
||||
private final MetadataWriteAdmissionCoordinator.TransactionPermit permit;
|
||||
|
||||
private PermitReleaseSynchronization(MetadataWriteAdmissionCoordinator.TransactionPermit permit) {
|
||||
this.permit = permit;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterCompletion(int status) {
|
||||
TransactionSynchronizationManager.unbindResourceIfPossible(resourceKey);
|
||||
permit.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.entity.alerter;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
class AlertEntitySerializationTest {
|
||||
|
||||
@Test
|
||||
void shouldOmitNullMapEntriesFromAlertApiPayloads() {
|
||||
Map<String, String> labels = new HashMap<>();
|
||||
labels.put("alertname", "CollectorUnavailable");
|
||||
labels.put("collectorVersion", null);
|
||||
SingleAlert singleAlert = SingleAlert.builder()
|
||||
.labels(labels)
|
||||
.annotations(new HashMap<>(labels))
|
||||
.build();
|
||||
GroupAlert groupAlert = GroupAlert.builder()
|
||||
.groupLabels(new HashMap<>(labels))
|
||||
.commonLabels(new HashMap<>(labels))
|
||||
.commonAnnotations(new HashMap<>(labels))
|
||||
.alerts(List.of(singleAlert))
|
||||
.build();
|
||||
|
||||
JsonNode payload = JsonUtil.fromJson(JsonUtil.toJson(groupAlert));
|
||||
|
||||
assertStringMapWithoutNullEntry(payload.path("groupLabels"));
|
||||
assertStringMapWithoutNullEntry(payload.path("commonLabels"));
|
||||
assertStringMapWithoutNullEntry(payload.path("commonAnnotations"));
|
||||
assertStringMapWithoutNullEntry(payload.path("alerts").path(0).path("labels"));
|
||||
assertStringMapWithoutNullEntry(payload.path("alerts").path(0).path("annotations"));
|
||||
}
|
||||
|
||||
private void assertStringMapWithoutNullEntry(JsonNode map) {
|
||||
assertEquals("CollectorUnavailable", map.path("alertname").textValue());
|
||||
assertFalse(map.has("collectorVersion"));
|
||||
}
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.runtime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
class NormalBusinessRuntimeConditionTest {
|
||||
|
||||
private final ApplicationContextRunner runner = new ApplicationContextRunner()
|
||||
.withUserConfiguration(ConditionalConfiguration.class);
|
||||
|
||||
@Test
|
||||
void fullSetupGatedContextStartsWithoutBusinessSideEffectBean() {
|
||||
runner.withPropertyValues(RuntimeMode.PROPERTY_NAME + "=full_setup_gated").run(context -> {
|
||||
assertTrue(context.isRunning());
|
||||
assertFalse(context.containsBean("businessSideEffect"));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalContextRegistersBusinessSideEffectBean() {
|
||||
runner.withPropertyValues(RuntimeMode.PROPERTY_NAME + "=normal").run(context ->
|
||||
assertTrue(context.containsBean("businessSideEffect")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingRuntimeModePreservesExistingNormalStartup() {
|
||||
runner.run(context -> assertTrue(context.containsBean("businessSideEffect")));
|
||||
}
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
static class ConditionalConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
String businessSideEffect() {
|
||||
return "started";
|
||||
}
|
||||
}
|
||||
}
|
||||
+504
@@ -0,0 +1,504 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import jakarta.persistence.Entity;
|
||||
import jakarta.persistence.GeneratedValue;
|
||||
import jakarta.persistence.Id;
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.springframework.aop.support.AopUtils;
|
||||
import org.springframework.aop.framework.Advised;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.transaction.IllegalTransactionStateException;
|
||||
import org.springframework.transaction.interceptor.BeanFactoryTransactionAttributeSourceAdvisor;
|
||||
import org.springframework.transaction.interceptor.TransactionInterceptor;
|
||||
import org.springframework.transaction.support.TransactionSynchronization;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
@ContextConfiguration(classes = MetadataWriteAdmissionIntegrationTest.TestConfiguration.class)
|
||||
class MetadataWriteAdmissionIntegrationTest {
|
||||
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
@Autowired
|
||||
private MetadataWriteAdmissionCoordinator coordinator;
|
||||
|
||||
@Autowired
|
||||
private AdmissionService service;
|
||||
|
||||
@Autowired
|
||||
private AdmissionRepository repository;
|
||||
|
||||
@Autowired
|
||||
private MetadataWriteAdmissionAdvisor admissionAdvisor;
|
||||
|
||||
@Autowired
|
||||
private BeanFactoryTransactionAttributeSourceAdvisor transactionAdvisor;
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutor() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void clearRows() {
|
||||
repository.deleteAll();
|
||||
}
|
||||
|
||||
@Test
|
||||
void drainWaitsForAdmittedTransactionCommitAndRollbackCompletion() throws Exception {
|
||||
CountDownLatch commitStarted = new CountDownLatch(1);
|
||||
CountDownLatch finishCommit = new CountDownLatch(1);
|
||||
CountDownLatch afterCompletionEntered = new CountDownLatch(1);
|
||||
CountDownLatch finishAfterCompletion = new CountDownLatch(1);
|
||||
Future<?> write = executor.submit(() -> service.holdWrite(
|
||||
commitStarted, finishCommit, afterCompletionEntered, finishAfterCompletion, false));
|
||||
assertThat(commitStarted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
|
||||
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
|
||||
() -> coordinator.acquire("operation-commit", Duration.ofSeconds(3)));
|
||||
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
|
||||
assertThat(acquiring.isDone()).isFalse();
|
||||
finishCommit.countDown();
|
||||
assertThat(afterCompletionEntered.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
|
||||
assertThat(acquiring.isDone()).isFalse();
|
||||
finishAfterCompletion.countDown();
|
||||
write.get(2, TimeUnit.SECONDS);
|
||||
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
|
||||
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.ACTIVE);
|
||||
}
|
||||
assertThat(service.count()).isEqualTo(1);
|
||||
|
||||
CountDownLatch rollbackStarted = new CountDownLatch(1);
|
||||
CountDownLatch finishRollback = new CountDownLatch(1);
|
||||
CountDownLatch rollbackCompletionEntered = new CountDownLatch(1);
|
||||
CountDownLatch finishRollbackCompletion = new CountDownLatch(1);
|
||||
Future<?> rollback = executor.submit(() -> service.holdWrite(
|
||||
rollbackStarted, finishRollback, rollbackCompletionEntered, finishRollbackCompletion, true));
|
||||
assertThat(rollbackStarted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
Future<MetadataWriteMaintenanceLease> rollbackDrain = executor.submit(
|
||||
() -> coordinator.acquire("operation-rollback", Duration.ofSeconds(3)));
|
||||
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
|
||||
finishRollback.countDown();
|
||||
assertThat(rollbackCompletionEntered.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
|
||||
assertThat(rollbackDrain.isDone()).isFalse();
|
||||
finishRollbackCompletion.countDown();
|
||||
assertThatThrownBy(() -> rollback.get(2, TimeUnit.SECONDS)).isInstanceOf(ExecutionException.class);
|
||||
try (MetadataWriteMaintenanceLease ignored = rollbackDrain.get(2, TimeUnit.SECONDS)) {
|
||||
assertThat(service.count()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void drainingAndActiveRejectNewWritesButAllowReadOnlyAndRepositoryTransactions() throws Exception {
|
||||
CountDownLatch admitted = new CountDownLatch(1);
|
||||
CountDownLatch finish = new CountDownLatch(1);
|
||||
Future<?> existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false));
|
||||
assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
|
||||
() -> coordinator.acquire("operation-gate", Duration.ofSeconds(3)));
|
||||
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
|
||||
|
||||
assertMaintenanceRejection(() -> service.write("draining"));
|
||||
assertThat(service.count()).isZero();
|
||||
finish.countDown();
|
||||
existing.get(2, TimeUnit.SECONDS);
|
||||
|
||||
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
|
||||
assertMaintenanceRejection(() -> service.write("active"));
|
||||
assertMaintenanceRejection(() -> repository.save(new AdmissionRow("repository")));
|
||||
assertMaintenanceRejection(() -> repository.deleteById(1L));
|
||||
assertThat(repository.count()).isEqualTo(1);
|
||||
assertThat(repository.findAll()).hasSize(1);
|
||||
assertThat(repository.declaredReadOnly()).hasSize(1);
|
||||
assertMaintenanceRejection(repository::declaredWritable);
|
||||
assertMaintenanceRejection(service::readOnlyThenNestedWrite);
|
||||
}
|
||||
service.write("open-again");
|
||||
assertThat(service.count()).isEqualTo(2);
|
||||
}
|
||||
|
||||
@Test
|
||||
void nestedWritableBoundaryReusesOuterThreadPermitWhileDrainStarts() throws Exception {
|
||||
CountDownLatch outerAdmitted = new CountDownLatch(1);
|
||||
CountDownLatch invokeNested = new CountDownLatch(1);
|
||||
Future<?> outer = executor.submit(() -> service.outerThenNested(outerAdmitted, invokeNested));
|
||||
assertThat(outerAdmitted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
|
||||
() -> coordinator.acquire("operation-nested", Duration.ofSeconds(3)));
|
||||
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
|
||||
invokeNested.countDown();
|
||||
outer.get(2, TimeUnit.SECONDS);
|
||||
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
|
||||
assertThat(repository.count()).isEqualTo(2);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void requiredWriteJoinedToReadOnlyOuterTransactionKeepsPermitUntilPhysicalCompletion() throws Exception {
|
||||
CountDownLatch nestedReturned = new CountDownLatch(1);
|
||||
CountDownLatch finishOuter = new CountDownLatch(1);
|
||||
Future<?> outer = executor.submit(() -> service.readOnlyOuterHoldingAfterNestedWrite(
|
||||
nestedReturned, finishOuter));
|
||||
assertThat(nestedReturned.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
|
||||
|
||||
Future<MetadataWriteMaintenanceLease> acquiring = executor.submit(
|
||||
() -> coordinator.acquire("operation-read-only-outer", Duration.ofSeconds(3)));
|
||||
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
|
||||
assertThat(acquiring.isDone()).isFalse();
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
|
||||
|
||||
finishOuter.countDown();
|
||||
outer.get(2, TimeUnit.SECONDS);
|
||||
try (MetadataWriteMaintenanceLease ignored = acquiring.get(2, TimeUnit.SECONDS)) {
|
||||
assertThat(repository.count()).isEqualTo(1);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void timeoutRestoresOpenAndLeaseEpochPreventsStaleOrConcurrentRelease() throws Exception {
|
||||
CountDownLatch admitted = new CountDownLatch(1);
|
||||
CountDownLatch finish = new CountDownLatch(1);
|
||||
Future<?> existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false));
|
||||
assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
|
||||
assertThatThrownBy(() -> coordinator.acquire("operation-timeout", Duration.ofMillis(100)))
|
||||
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class,
|
||||
error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.DRAIN_TIMEOUT));
|
||||
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
|
||||
service.write("accepted-after-timeout");
|
||||
finish.countDown();
|
||||
existing.get(2, TimeUnit.SECONDS);
|
||||
|
||||
MetadataWriteMaintenanceLease stale = coordinator.acquire("operation-one", Duration.ofSeconds(1));
|
||||
assertConflict(() -> coordinator.acquire("operation-one", Duration.ofSeconds(1)));
|
||||
assertConflict(() -> coordinator.acquire("operation-other", Duration.ofSeconds(1)));
|
||||
stale.close();
|
||||
MetadataWriteMaintenanceLease current = coordinator.acquire("operation-two", Duration.ofSeconds(1));
|
||||
stale.close();
|
||||
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.ACTIVE);
|
||||
assertThat(coordinator.snapshot().operationId()).isEqualTo("operation-two");
|
||||
current.close();
|
||||
current.close();
|
||||
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
|
||||
}
|
||||
|
||||
@Test
|
||||
void interruptedDrainReopensAdmissionAndTransactionStartFailureDoesNotLeakPermit() throws Exception {
|
||||
assertThatThrownBy(service::mandatoryWrite)
|
||||
.isInstanceOf(IllegalTransactionStateException.class);
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isZero();
|
||||
|
||||
CountDownLatch admitted = new CountDownLatch(1);
|
||||
CountDownLatch finish = new CountDownLatch(1);
|
||||
Future<?> existing = executor.submit(() -> service.holdWrite(admitted, finish, null, null, false));
|
||||
assertThat(admitted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
AtomicReference<MetadataWriteAdmissionException> failure = new AtomicReference<>();
|
||||
CountDownLatch interrupted = new CountDownLatch(1);
|
||||
Future<?> acquiring = executor.submit(() -> {
|
||||
try {
|
||||
coordinator.acquire("operation-interrupted", Duration.ofSeconds(30));
|
||||
} catch (MetadataWriteAdmissionException exception) {
|
||||
failure.set(exception);
|
||||
} finally {
|
||||
interrupted.countDown();
|
||||
}
|
||||
});
|
||||
awaitPhase(MetadataWriteAdmissionPhase.DRAINING);
|
||||
acquiring.cancel(true);
|
||||
assertThat(interrupted.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
assertThat(failure.get().code()).isEqualTo(MetadataWriteAdmissionErrorCode.ACQUISITION_INTERRUPTED);
|
||||
assertThat(coordinator.snapshot().phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isEqualTo(1);
|
||||
finish.countDown();
|
||||
existing.get(2, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatesOperationAndDurationWithoutLeakingRawFailures() {
|
||||
assertInvalid(() -> coordinator.acquire(" ", Duration.ofSeconds(1)));
|
||||
assertInvalid(() -> coordinator.acquire("operation", null));
|
||||
assertInvalid(() -> coordinator.acquire("operation", Duration.ofSeconds(-1)));
|
||||
assertInvalid(() -> coordinator.acquire("operation", Duration.ofSeconds(Long.MAX_VALUE)));
|
||||
|
||||
MetadataWriteMaintenanceLease lease = coordinator.acquire("zero-wait", Duration.ZERO);
|
||||
lease.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
void admissionAdvisorUsesTransactionAttributesAndRunsOutsideTransactionInterceptor() throws Exception {
|
||||
assertThat(((Advised) service).getAdvisors()).contains(admissionAdvisor);
|
||||
assertThat(admissionAdvisor.getOrder()).isLessThan(transactionAdvisor.getOrder());
|
||||
assertThat(admissionAdvisor.getPointcut().getMethodMatcher().matches(
|
||||
AdmissionService.class.getMethod("write", String.class), AdmissionService.class)).isTrue();
|
||||
assertThat(AopUtils.isAopProxy(repository)).isTrue();
|
||||
Advised repositoryProxy = (Advised) repository;
|
||||
assertThat(AopUtils.isAopProxy(repositoryProxy.getTargetSource().getTarget())).isFalse();
|
||||
assertThat(Arrays.stream(repositoryProxy.getAdvisors())
|
||||
.filter(MetadataWriteAdmissionAdvisor.class::isInstance)).hasSize(1);
|
||||
assertThat(Arrays.stream(repositoryProxy.getAdvisors())
|
||||
.filter(advisor -> advisor.getAdvice() instanceof TransactionInterceptor)).hasSize(1);
|
||||
|
||||
MetadataWriteAdmissionSnapshot observed = service.observeAdmissionInsideTransaction();
|
||||
assertThat(observed.phase()).isEqualTo(MetadataWriteAdmissionPhase.OPEN);
|
||||
assertThat(observed.activeWritableTransactions()).isEqualTo(1);
|
||||
}
|
||||
|
||||
private void awaitPhase(MetadataWriteAdmissionPhase phase) throws InterruptedException {
|
||||
long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);
|
||||
while (coordinator.snapshot().phase() != phase && System.nanoTime() < deadline) {
|
||||
Thread.onSpinWait();
|
||||
}
|
||||
assertThat(coordinator.snapshot().phase()).isEqualTo(phase);
|
||||
}
|
||||
|
||||
private void assertMaintenanceRejection(Runnable write) {
|
||||
assertThatThrownBy(write::run)
|
||||
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class, error -> {
|
||||
assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.MAINTENANCE_ACTIVE);
|
||||
assertThat(error.getMessage()).isEqualTo("Metadata writes are temporarily unavailable");
|
||||
assertThat(error.getCause()).isNull();
|
||||
});
|
||||
}
|
||||
|
||||
private void assertConflict(ThrowingAcquire acquire) {
|
||||
assertThatThrownBy(acquire::run)
|
||||
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class,
|
||||
error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.OPERATION_CONFLICT));
|
||||
}
|
||||
|
||||
private void assertInvalid(ThrowingAcquire acquire) {
|
||||
assertThatThrownBy(acquire::run)
|
||||
.isInstanceOfSatisfying(MetadataWriteAdmissionException.class,
|
||||
error -> assertThat(error.code()).isEqualTo(MetadataWriteAdmissionErrorCode.INVALID_REQUEST));
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface ThrowingAcquire {
|
||||
void run();
|
||||
}
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@EnableTransactionManagement(order = 200)
|
||||
@EnableJpaRepositories(considerNestedRepositories = true,
|
||||
basePackageClasses = MetadataWriteAdmissionIntegrationTest.class)
|
||||
@Import(MetadataWriteAdmissionConfiguration.class)
|
||||
static class TestConfiguration {
|
||||
|
||||
@Bean
|
||||
DriverManagerDataSource dataSource() {
|
||||
return new DriverManagerDataSource("jdbc:h2:mem:metadata-admission;DB_CLOSE_DELAY=-1", "sa", "");
|
||||
}
|
||||
|
||||
@Bean
|
||||
LocalContainerEntityManagerFactoryBean entityManagerFactory(DriverManagerDataSource dataSource) {
|
||||
LocalContainerEntityManagerFactoryBean factory = new LocalContainerEntityManagerFactoryBean();
|
||||
factory.setDataSource(dataSource);
|
||||
factory.setPackagesToScan(AdmissionRow.class.getPackageName());
|
||||
factory.setJpaVendorAdapter(new HibernateJpaVendorAdapter());
|
||||
factory.setJpaPropertyMap(Map.of("hibernate.hbm2ddl.auto", "create-drop"));
|
||||
return factory;
|
||||
}
|
||||
|
||||
@Bean
|
||||
PlatformTransactionManager transactionManager(jakarta.persistence.EntityManagerFactory factory) {
|
||||
return new JpaTransactionManager(factory);
|
||||
}
|
||||
|
||||
@Bean
|
||||
AdmissionNestedWriter nestedWriter(AdmissionRepository repository) {
|
||||
return new AdmissionNestedWriter(repository);
|
||||
}
|
||||
|
||||
@Bean
|
||||
AdmissionService admissionService(
|
||||
AdmissionRepository repository,
|
||||
AdmissionNestedWriter nestedWriter,
|
||||
MetadataWriteAdmissionCoordinator coordinator) {
|
||||
return new AdmissionService(repository, nestedWriter, coordinator);
|
||||
}
|
||||
}
|
||||
|
||||
@Entity(name = "AdmissionRow")
|
||||
static class AdmissionRow {
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
private Long id;
|
||||
|
||||
private String label;
|
||||
|
||||
protected AdmissionRow() {
|
||||
}
|
||||
|
||||
AdmissionRow(String value) {
|
||||
this.label = value;
|
||||
}
|
||||
}
|
||||
|
||||
interface AdmissionRepository extends JpaRepository<AdmissionRow, Long> {
|
||||
|
||||
@Override
|
||||
<S extends AdmissionRow> S save(S entity);
|
||||
|
||||
@Override
|
||||
void deleteById(Long id);
|
||||
|
||||
@Query("select row from AdmissionRow row")
|
||||
@Transactional(readOnly = true)
|
||||
List<AdmissionRow> declaredReadOnly();
|
||||
|
||||
@Query("select row from AdmissionRow row")
|
||||
@Transactional
|
||||
List<AdmissionRow> declaredWritable();
|
||||
}
|
||||
|
||||
static class AdmissionNestedWriter {
|
||||
|
||||
private final AdmissionRepository repository;
|
||||
|
||||
AdmissionNestedWriter(AdmissionRepository repository) {
|
||||
this.repository = repository;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void write() {
|
||||
repository.saveAndFlush(new AdmissionRow("nested"));
|
||||
}
|
||||
}
|
||||
|
||||
static class AdmissionService {
|
||||
|
||||
private final AdmissionRepository repository;
|
||||
private final AdmissionNestedWriter nestedWriter;
|
||||
private final MetadataWriteAdmissionCoordinator coordinator;
|
||||
|
||||
AdmissionService(
|
||||
AdmissionRepository repository,
|
||||
AdmissionNestedWriter nestedWriter,
|
||||
MetadataWriteAdmissionCoordinator coordinator) {
|
||||
this.repository = repository;
|
||||
this.nestedWriter = nestedWriter;
|
||||
this.coordinator = coordinator;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void write(String value) {
|
||||
repository.saveAndFlush(new AdmissionRow(value));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void holdWrite(
|
||||
CountDownLatch admitted,
|
||||
CountDownLatch finish,
|
||||
CountDownLatch afterCompletionEntered,
|
||||
CountDownLatch finishAfterCompletion,
|
||||
boolean rollback) {
|
||||
repository.saveAndFlush(new AdmissionRow("held"));
|
||||
if (afterCompletionEntered != null) {
|
||||
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
|
||||
@Override
|
||||
public void afterCompletion(int status) {
|
||||
afterCompletionEntered.countDown();
|
||||
await(finishAfterCompletion);
|
||||
}
|
||||
});
|
||||
}
|
||||
admitted.countDown();
|
||||
await(finish);
|
||||
if (rollback) {
|
||||
throw new IllegalStateException("rollback requested");
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void outerThenNested(CountDownLatch admitted, CountDownLatch invokeNested) {
|
||||
repository.saveAndFlush(new AdmissionRow("outer"));
|
||||
admitted.countDown();
|
||||
await(invokeNested);
|
||||
nestedWriter.write();
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public long count() {
|
||||
return repository.count();
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public void readOnlyThenNestedWrite() {
|
||||
nestedWriter.write();
|
||||
}
|
||||
|
||||
@Transactional(readOnly = true)
|
||||
public void readOnlyOuterHoldingAfterNestedWrite(
|
||||
CountDownLatch nestedReturned, CountDownLatch finishOuter) {
|
||||
nestedWriter.write();
|
||||
nestedReturned.countDown();
|
||||
await(finishOuter);
|
||||
}
|
||||
|
||||
@Transactional(propagation = Propagation.MANDATORY)
|
||||
public void mandatoryWrite() {
|
||||
repository.saveAndFlush(new AdmissionRow("mandatory"));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public MetadataWriteAdmissionSnapshot observeAdmissionInsideTransaction() {
|
||||
assertThat(TransactionSynchronizationManager.isActualTransactionActive()).isTrue();
|
||||
return coordinator.snapshot();
|
||||
}
|
||||
|
||||
private static void await(CountDownLatch latch) {
|
||||
try {
|
||||
if (!latch.await(2, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("test latch timed out");
|
||||
}
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("test interrupted", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.common.transaction;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.transaction.support.TransactionSynchronizationManager;
|
||||
|
||||
class TransactionCompletionPermitRegistryTest {
|
||||
|
||||
@AfterEach
|
||||
void clearTransactionState() {
|
||||
TransactionSynchronizationManager.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
void synchronizationRegistrationFailureReleasesPermitAndResource() {
|
||||
MetadataWriteAdmissionCoordinator coordinator = new MetadataWriteAdmissionCoordinator();
|
||||
TransactionCompletionPermitRegistry registry = new TransactionCompletionPermitRegistry(synchronization -> {
|
||||
throw new IllegalStateException("registration failed");
|
||||
});
|
||||
TransactionSynchronizationManager.initSynchronization();
|
||||
TransactionSynchronizationManager.setActualTransactionActive(true);
|
||||
MetadataWriteAdmissionCoordinator.TransactionPermit permit = coordinator.admitWritableTransaction();
|
||||
|
||||
assertThatThrownBy(() -> registry.bind(permit)).isInstanceOf(IllegalStateException.class);
|
||||
|
||||
assertThat(registry.hasPermit()).isFalse();
|
||||
assertThat(coordinator.snapshot().activeWritableTransactions()).isZero();
|
||||
}
|
||||
}
|
||||
@@ -76,6 +76,12 @@ resourceRole:
|
||||
- /api/notice/**===post===[admin,user]
|
||||
- /api/notice/**===put===[admin,user]
|
||||
- /api/notice/**===delete===[admin]
|
||||
- /api/config/deployment===get===[admin]
|
||||
- /api/config/deployment/validate===post===[admin]
|
||||
- /api/config/deployment/metadata-migrations===post===[admin]
|
||||
- /api/config/deployment/metadata-migrations/*===get===[admin]
|
||||
- /api/config/deployment/metadata-migrations/*/activate===post===[admin]
|
||||
- /api/config/deployment/metadata-migrations/*/export===post===[admin]
|
||||
- /api/config/email===get===[admin,user,guest]
|
||||
- /api/config/email===post===[admin]
|
||||
- /api/config/sms===get===[admin,user,guest]
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
package org.apache.hertzbeat.grafana.config;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.grafana.service.DatasourceService;
|
||||
import org.apache.hertzbeat.grafana.service.ServiceAccountService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -28,6 +29,7 @@ import org.springframework.stereotype.Component;
|
||||
* grafana init
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Slf4j
|
||||
public class GrafanaInit implements CommandLineRunner {
|
||||
@Autowired
|
||||
|
||||
@@ -56,24 +56,35 @@ public class LogSseManager {
|
||||
|
||||
private final Map<Long, SseSubscriber> emitters = new ConcurrentHashMap<>();
|
||||
private final Queue<LogEntry> logQueue = new ConcurrentLinkedQueue<>();
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r, "sse-batch-scheduler");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
private final ExecutorService senderPool = Executors.newCachedThreadPool(r -> {
|
||||
Thread t = new Thread(r, "sse-sender");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
private final AtomicLong queueSize = new AtomicLong(0);
|
||||
private ScheduledExecutorService scheduler;
|
||||
private ExecutorService senderPool;
|
||||
|
||||
public LogSseManager() {
|
||||
scheduler.scheduleAtFixedRate(this::flushBatch, BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS);
|
||||
synchronized void start() {
|
||||
if (scheduler != null) {
|
||||
return;
|
||||
}
|
||||
scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = new Thread(r, "sse-batch-scheduler");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
senderPool = Executors.newCachedThreadPool(r -> {
|
||||
Thread t = new Thread(r, "sse-sender");
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
});
|
||||
ExecutorService currentSenderPool = senderPool;
|
||||
scheduler.scheduleAtFixedRate(
|
||||
() -> flushBatch(currentSenderPool), BATCH_INTERVAL_MS, BATCH_INTERVAL_MS, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
private final AtomicLong queueSize = new AtomicLong(0);
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
public synchronized void shutdown() {
|
||||
if (scheduler == null) {
|
||||
return;
|
||||
}
|
||||
scheduler.shutdown();
|
||||
senderPool.shutdown();
|
||||
try {
|
||||
@@ -84,6 +95,8 @@ public class LogSseManager {
|
||||
}
|
||||
scheduler.shutdownNow();
|
||||
senderPool.shutdownNow();
|
||||
scheduler = null;
|
||||
senderPool = null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -117,7 +130,13 @@ public class LogSseManager {
|
||||
/**
|
||||
* Flush queued logs to all subscribers in batch
|
||||
*/
|
||||
private void flushBatch() {
|
||||
synchronized void flushBatch() {
|
||||
if (senderPool != null) {
|
||||
flushBatch(senderPool);
|
||||
}
|
||||
}
|
||||
|
||||
private void flushBatch(ExecutorService currentSenderPool) {
|
||||
try {
|
||||
if (logQueue.isEmpty() || emitters.isEmpty()) {
|
||||
return;
|
||||
@@ -140,7 +159,7 @@ public class LogSseManager {
|
||||
SseSubscriber subscriber = e.getValue();
|
||||
List<LogEntry> filtered = filterLogs(batch, subscriber.filters);
|
||||
if (!filtered.isEmpty()) {
|
||||
senderPool.submit(() -> sendToSubscriber(clientId, subscriber.emitter, filtered));
|
||||
currentSenderPool.submit(() -> sendToSubscriber(clientId, subscriber.emitter, filtered));
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.log.notice;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Starts log SSE delivery threads only in normal runtime. */
|
||||
@Component("legacyLogSseManagerLifecycle")
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class LogSseManagerLifecycle implements CommandLineRunner {
|
||||
|
||||
private final LogSseManager manager;
|
||||
|
||||
public LogSseManagerLifecycle(LogSseManager manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
manager.start();
|
||||
}
|
||||
}
|
||||
@@ -51,6 +51,7 @@ class LogSseManagerTest {
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
logSseManager = new LogSseManager();
|
||||
logSseManager.start();
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
@@ -172,6 +173,17 @@ class LogSseManagerTest {
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void flushAfterShutdownIsSafeNoOp() {
|
||||
logSseManager.shutdown();
|
||||
logSseManager.broadcast(createLogEntry("INFO", "after shutdown"));
|
||||
|
||||
logSseManager.flushBatch();
|
||||
logSseManager.shutdown();
|
||||
|
||||
assertEquals(1, logSseManager.getQueueSize());
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to create a subscriber and inject a mock emitter for testing
|
||||
*/
|
||||
@@ -190,4 +202,4 @@ class LogSseManagerTest {
|
||||
.body(body)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,7 +90,6 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-observability</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- spring -->
|
||||
<dependency>
|
||||
@@ -152,6 +151,11 @@
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<!-- email -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
@@ -172,6 +176,10 @@
|
||||
<groupId>com.usthe.sureness</groupId>
|
||||
<artifactId>spring-boot3-starter-sureness</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.security</groupId>
|
||||
<artifactId>spring-security-crypto</artifactId>
|
||||
</dependency>
|
||||
<!-- okhttp -->
|
||||
<dependency>
|
||||
<groupId>com.squareup.okhttp3</groupId>
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.component.sd;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.locks.Condition;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase;
|
||||
|
||||
/** Admission and drain state for the single service-discovery consumer loop. */
|
||||
final class ServiceDiscoveryMaintenanceGate {
|
||||
|
||||
private final ReentrantLock lock = new ReentrantLock();
|
||||
private final Condition stateChanged = lock.newCondition();
|
||||
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
|
||||
private Thread pollingThread;
|
||||
private boolean polling;
|
||||
private boolean processing;
|
||||
private boolean maintenanceWakeup;
|
||||
private boolean terminal;
|
||||
|
||||
void beforePoll() throws InterruptedException {
|
||||
lock.lockInterruptibly();
|
||||
try {
|
||||
while (phase != MetadataMaintenancePhase.RUNNING && !terminal) {
|
||||
stateChanged.await();
|
||||
}
|
||||
if (terminal) {
|
||||
throw new InterruptedException();
|
||||
}
|
||||
polling = true;
|
||||
pollingThread = Thread.currentThread();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/** Finish acquisition and promote a returned message to in-flight work. */
|
||||
boolean pollCompleted(boolean messageReturned) {
|
||||
lock.lock();
|
||||
try {
|
||||
polling = false;
|
||||
pollingThread = null;
|
||||
if (messageReturned) {
|
||||
processing = true;
|
||||
}
|
||||
boolean clearMaintenanceInterrupt = maintenanceWakeup && !terminal;
|
||||
maintenanceWakeup = false;
|
||||
stateChanged.signalAll();
|
||||
return clearMaintenanceInterrupt;
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
boolean pollInterrupted() {
|
||||
lock.lock();
|
||||
try {
|
||||
polling = false;
|
||||
pollingThread = null;
|
||||
boolean expected = maintenanceWakeup && !terminal;
|
||||
maintenanceWakeup = false;
|
||||
stateChanged.signalAll();
|
||||
return expected;
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void workCompleted() {
|
||||
lock.lock();
|
||||
try {
|
||||
processing = false;
|
||||
stateChanged.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void quiesce(Duration timeout) {
|
||||
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
|
||||
try {
|
||||
lock.lockInterruptibly();
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw MetadataMaintenanceException.quiesceInterrupted();
|
||||
}
|
||||
try {
|
||||
if (phase == MetadataMaintenancePhase.QUIESCED) {
|
||||
return;
|
||||
}
|
||||
if (phase == MetadataMaintenancePhase.RUNNING) {
|
||||
phase = MetadataMaintenancePhase.QUIESCING;
|
||||
if (pollingThread != null) {
|
||||
maintenanceWakeup = true;
|
||||
pollingThread.interrupt();
|
||||
}
|
||||
}
|
||||
while (polling || processing) {
|
||||
long remainingNanos = deadline.remainingNanos();
|
||||
if (remainingNanos <= 0) {
|
||||
reopen();
|
||||
throw MetadataMaintenanceException.quiesceTimeout();
|
||||
}
|
||||
try {
|
||||
stateChanged.awaitNanos(remainingNanos);
|
||||
} catch (InterruptedException exception) {
|
||||
reopen();
|
||||
Thread.currentThread().interrupt();
|
||||
throw MetadataMaintenanceException.quiesceInterrupted();
|
||||
}
|
||||
}
|
||||
phase = MetadataMaintenancePhase.QUIESCED;
|
||||
stateChanged.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void resume() {
|
||||
lock.lock();
|
||||
try {
|
||||
if (terminal) {
|
||||
return;
|
||||
}
|
||||
if (phase != MetadataMaintenancePhase.RUNNING) {
|
||||
reopen();
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void stop() {
|
||||
lock.lock();
|
||||
try {
|
||||
if (terminal) {
|
||||
return;
|
||||
}
|
||||
terminal = true;
|
||||
if (pollingThread != null) {
|
||||
pollingThread.interrupt();
|
||||
}
|
||||
stateChanged.signalAll();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
MetadataMaintenancePhase phase() {
|
||||
lock.lock();
|
||||
try {
|
||||
return phase;
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void reopen() {
|
||||
phase = MetadataMaintenancePhase.RUNNING;
|
||||
stateChanged.signalAll();
|
||||
}
|
||||
}
|
||||
+76
-13
@@ -18,6 +18,14 @@
|
||||
package org.apache.hertzbeat.manager.component.sd;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
@@ -28,31 +36,31 @@ import org.apache.hertzbeat.common.entity.manager.Param;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.common.util.BackoffUtils;
|
||||
import org.apache.hertzbeat.common.util.ExponentialBackoff;
|
||||
import org.apache.hertzbeat.manager.dao.CollectorMonitorBindDao;
|
||||
import org.apache.hertzbeat.manager.dao.MonitorBindDao;
|
||||
import org.apache.hertzbeat.manager.dao.MonitorDao;
|
||||
import org.apache.hertzbeat.manager.dao.ParamDao;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase;
|
||||
import org.apache.hertzbeat.manager.scheduler.ManagerWorkerPool;
|
||||
import org.apache.hertzbeat.manager.service.MonitorService;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Service Discovery Worker
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class ServiceDiscoveryWorker implements InitializingBean {
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Order(100)
|
||||
public class ServiceDiscoveryWorker
|
||||
implements InitializingBean, DisposableBean, MetadataMaintenanceParticipant {
|
||||
|
||||
private static final String FILED_HOST = "host";
|
||||
private static final String FILED_PORT = "port";
|
||||
@@ -63,6 +71,7 @@ public class ServiceDiscoveryWorker implements InitializingBean {
|
||||
private final CollectorMonitorBindDao collectorMonitorBindDao;
|
||||
private final CommonDataQueue dataQueue;
|
||||
private final ManagerWorkerPool workerPool;
|
||||
private final ServiceDiscoveryMaintenanceGate maintenanceGate = new ServiceDiscoveryMaintenanceGate();
|
||||
|
||||
public ServiceDiscoveryWorker(MonitorService monitorService, ParamDao paramDao, MonitorDao monitorDao,
|
||||
MonitorBindDao monitorBindDao, CollectorMonitorBindDao collectorMonitorBindDao,
|
||||
@@ -81,12 +90,65 @@ public class ServiceDiscoveryWorker implements InitializingBean {
|
||||
workerPool.executeLongRunning(new SdUpdateTask());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String participantId() {
|
||||
return "service-discovery";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void quiesce(Duration timeout) {
|
||||
maintenanceGate.quiesce(timeout);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void resume() {
|
||||
maintenanceGate.resume();
|
||||
}
|
||||
|
||||
MetadataMaintenancePhase maintenancePhase() {
|
||||
return maintenanceGate.phase();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
maintenanceGate.stop();
|
||||
}
|
||||
|
||||
private class SdUpdateTask implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
try (final CollectRep.MetricsData metricsData = dataQueue.pollServiceDiscoveryData()) {
|
||||
CollectRep.MetricsData polledData;
|
||||
try {
|
||||
maintenanceGate.beforePoll();
|
||||
polledData = dataQueue.pollServiceDiscoveryData();
|
||||
} catch (InterruptedException interruptedException) {
|
||||
if (maintenanceGate.pollInterrupted()) {
|
||||
Thread.interrupted();
|
||||
continue;
|
||||
}
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
} catch (RuntimeException exception) {
|
||||
boolean clearMaintenanceInterrupt = maintenanceGate.pollCompleted(false);
|
||||
if (clearMaintenanceInterrupt) {
|
||||
Thread.interrupted();
|
||||
}
|
||||
if (exception instanceof CommonDataQueueUnknownException) {
|
||||
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
log.error(exception.getMessage(), exception);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
boolean clearMaintenanceInterrupt = maintenanceGate.pollCompleted(polledData != null);
|
||||
if (clearMaintenanceInterrupt) {
|
||||
Thread.interrupted();
|
||||
}
|
||||
try (final CollectRep.MetricsData metricsData = polledData) {
|
||||
if (metricsData == null) {
|
||||
continue;
|
||||
}
|
||||
@@ -163,15 +225,16 @@ public class ServiceDiscoveryWorker implements InitializingBean {
|
||||
final Set<Long> needCancelMonitorIdSet = subMonitorBindMap.values().stream()
|
||||
.map(MonitorBind::getMonitorId).collect(Collectors.toSet());
|
||||
monitorService.deleteMonitors(needCancelMonitorIdSet);
|
||||
} catch (InterruptedException interruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
} catch (CommonDataQueueUnknownException ue) {
|
||||
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
|
||||
break;
|
||||
}
|
||||
} catch (Exception exception) {
|
||||
log.error(exception.getMessage(), exception);
|
||||
} finally {
|
||||
if (polledData != null) {
|
||||
maintenanceGate.workCompleted();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+155
-102
@@ -47,8 +47,13 @@ import org.apache.hertzbeat.manager.dao.MonitorDao;
|
||||
import org.apache.hertzbeat.manager.dao.StatusPageComponentDao;
|
||||
import org.apache.hertzbeat.manager.dao.StatusPageHistoryDao;
|
||||
import org.apache.hertzbeat.manager.dao.StatusPageOrgDao;
|
||||
import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceParticipant;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenancePhase;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.data.jpa.domain.Specification;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
@@ -57,7 +62,8 @@ import org.springframework.stereotype.Component;
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class CalculateStatus implements DisposableBean {
|
||||
@Order(200)
|
||||
public class CalculateStatus implements DisposableBean, MetadataMaintenanceParticipant {
|
||||
|
||||
private static final int DEFAULT_CALCULATE_INTERVAL_TIME = 300;
|
||||
|
||||
@@ -70,61 +76,145 @@ public class CalculateStatus implements DisposableBean {
|
||||
private final MonitorDao monitorDao;
|
||||
|
||||
private final int intervals;
|
||||
private final VirtualThreadProperties virtualThreadProperties;
|
||||
|
||||
private final ScheduledExecutorService calculateScheduler;
|
||||
private ScheduledExecutorService calculateScheduler;
|
||||
|
||||
private final ScheduledExecutorService combineHistoryScheduler;
|
||||
private ScheduledExecutorService combineHistoryScheduler;
|
||||
|
||||
private final ExecutorService calculateExecutor;
|
||||
private ExecutorService calculateExecutor;
|
||||
|
||||
private final ExecutorService combineHistoryExecutor;
|
||||
private ExecutorService combineHistoryExecutor;
|
||||
|
||||
private final ScheduledDispatchTask calculateTask;
|
||||
private PausableDispatchTask calculateTask;
|
||||
|
||||
private final ScheduledDispatchTask combineHistoryTask;
|
||||
private PausableDispatchTask combineHistoryTask;
|
||||
|
||||
private boolean started;
|
||||
|
||||
private volatile MetadataMaintenancePhase maintenancePhase = MetadataMaintenancePhase.RUNNING;
|
||||
|
||||
public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao,
|
||||
StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao,
|
||||
MonitorDao monitorDao) {
|
||||
this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao,
|
||||
VirtualThreadProperties.defaults(), true);
|
||||
VirtualThreadProperties.defaults());
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao,
|
||||
StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao,
|
||||
MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties) {
|
||||
this(statusPageOrgDao, statusPageComponentDao, statusProperties, statusPageHistoryDao, monitorDao,
|
||||
virtualThreadProperties, true);
|
||||
}
|
||||
|
||||
CalculateStatus(StatusPageOrgDao statusPageOrgDao, StatusPageComponentDao statusPageComponentDao,
|
||||
StatusProperties statusProperties, StatusPageHistoryDao statusPageHistoryDao,
|
||||
MonitorDao monitorDao, VirtualThreadProperties virtualThreadProperties, boolean autoStart) {
|
||||
this.statusPageOrgDao = statusPageOrgDao;
|
||||
this.monitorDao = monitorDao;
|
||||
this.statusPageComponentDao = statusPageComponentDao;
|
||||
this.statusPageHistoryDao = statusPageHistoryDao;
|
||||
intervals = statusProperties.getCalculate() == null ? DEFAULT_CALCULATE_INTERVAL_TIME : statusProperties.getCalculate().getInterval();
|
||||
this.calculateScheduler = createScheduler("status-page-calculate-%d", "Status calculate has uncaughtException.");
|
||||
this.combineHistoryScheduler = createScheduler("status-page-history-%d", "History combine has uncaughtException.");
|
||||
this.calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-",
|
||||
"Status calculate worker has uncaughtException.");
|
||||
this.combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-",
|
||||
"History combine worker has uncaughtException.");
|
||||
this.calculateTask = new ScheduledDispatchTask(calculateExecutor, this::runCalculate);
|
||||
this.combineHistoryTask = new ScheduledDispatchTask(combineHistoryExecutor, this::runCombineHistory);
|
||||
if (autoStart) {
|
||||
startCalculate();
|
||||
startCombineHistory();
|
||||
StatusProperties.CalculateProperties calculateProperties = statusProperties.getCalculate();
|
||||
intervals = calculateProperties == null
|
||||
? DEFAULT_CALCULATE_INTERVAL_TIME : calculateProperties.getInterval();
|
||||
this.virtualThreadProperties = virtualThreadProperties == null
|
||||
? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
}
|
||||
|
||||
synchronized void start() {
|
||||
if (started) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
calculateScheduler = createScheduler(
|
||||
"status-page-calculate-%d", "Status calculate has uncaughtException.");
|
||||
combineHistoryScheduler = createScheduler(
|
||||
"status-page-history-%d", "History combine has uncaughtException.");
|
||||
calculateExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-calculate-vt-",
|
||||
"Status calculate worker has uncaughtException.");
|
||||
combineHistoryExecutor = createVirtualExecutor(virtualThreadProperties, "status-page-history-vt-",
|
||||
"History combine worker has uncaughtException.");
|
||||
PausableDispatchTask currentCalculateTask =
|
||||
new PausableDispatchTask(calculateExecutor, this::runCalculate);
|
||||
PausableDispatchTask currentCombineHistoryTask =
|
||||
new PausableDispatchTask(combineHistoryExecutor, this::runCombineHistory);
|
||||
calculateTask = currentCalculateTask;
|
||||
combineHistoryTask = currentCombineHistoryTask;
|
||||
startCalculate(currentCalculateTask);
|
||||
startCombineHistory(currentCombineHistoryTask);
|
||||
started = true;
|
||||
} catch (RuntimeException | Error e) {
|
||||
destroy();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private void startCalculate() {
|
||||
calculateScheduler.scheduleAtFixedRate(this::dispatchCalculate, 5, intervals, TimeUnit.SECONDS);
|
||||
synchronized boolean isStarted() {
|
||||
return started;
|
||||
}
|
||||
|
||||
private void startCombineHistory() {
|
||||
@Override
|
||||
public String participantId() {
|
||||
return "status-calculation";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void quiesce(Duration timeout) {
|
||||
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
|
||||
PausableDispatchTask currentCalculateTask;
|
||||
PausableDispatchTask currentCombineHistoryTask;
|
||||
synchronized (this) {
|
||||
if (maintenancePhase == MetadataMaintenancePhase.QUIESCED) {
|
||||
return;
|
||||
}
|
||||
maintenancePhase = MetadataMaintenancePhase.QUIESCING;
|
||||
currentCalculateTask = calculateTask;
|
||||
currentCombineHistoryTask = combineHistoryTask;
|
||||
if (currentCalculateTask != null) {
|
||||
currentCalculateTask.pauseAdmission();
|
||||
}
|
||||
if (currentCombineHistoryTask != null) {
|
||||
currentCombineHistoryTask.pauseAdmission();
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (currentCalculateTask != null) {
|
||||
currentCalculateTask.awaitDrained(deadline);
|
||||
}
|
||||
if (currentCombineHistoryTask != null) {
|
||||
currentCombineHistoryTask.awaitDrained(deadline);
|
||||
}
|
||||
maintenancePhase = MetadataMaintenancePhase.QUIESCED;
|
||||
} catch (MetadataMaintenanceException exception) {
|
||||
resumeTasks(currentCalculateTask, currentCombineHistoryTask);
|
||||
maintenancePhase = MetadataMaintenancePhase.RUNNING;
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void resume() {
|
||||
if (maintenancePhase == MetadataMaintenancePhase.RUNNING) {
|
||||
return;
|
||||
}
|
||||
maintenancePhase = MetadataMaintenancePhase.RUNNING;
|
||||
resumeTasks(calculateTask, combineHistoryTask);
|
||||
}
|
||||
|
||||
MetadataMaintenancePhase maintenancePhase() {
|
||||
return maintenancePhase;
|
||||
}
|
||||
|
||||
private void resumeTasks(PausableDispatchTask first, PausableDispatchTask second) {
|
||||
if (second != null) {
|
||||
second.resumeAdmission();
|
||||
}
|
||||
if (first != null) {
|
||||
first.resumeAdmission();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void startCalculate(PausableDispatchTask currentCalculateTask) {
|
||||
calculateScheduler.scheduleAtFixedRate(currentCalculateTask::dispatch, 5, intervals, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private void startCombineHistory(PausableDispatchTask currentCombineHistoryTask) {
|
||||
// combine history every day at 1:00 AM
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
LocalDateTime nextRun = now.withHour(1).withMinute(0).withSecond(0);
|
||||
@@ -132,7 +222,7 @@ public class CalculateStatus implements DisposableBean {
|
||||
nextRun = nextRun.plusDays(1);
|
||||
}
|
||||
long delay = Duration.between(now, nextRun).toMillis();
|
||||
combineHistoryScheduler.scheduleAtFixedRate(this::dispatchCombineHistory, delay,
|
||||
combineHistoryScheduler.scheduleAtFixedRate(currentCombineHistoryTask::dispatch, delay,
|
||||
TimeUnit.DAYS.toMillis(1), TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@@ -145,23 +235,52 @@ public class CalculateStatus implements DisposableBean {
|
||||
}
|
||||
|
||||
void dispatchCalculate() {
|
||||
calculateTask.dispatch();
|
||||
PausableDispatchTask currentTask;
|
||||
synchronized (this) {
|
||||
currentTask = calculateTask;
|
||||
}
|
||||
if (currentTask != null) {
|
||||
currentTask.dispatch();
|
||||
}
|
||||
}
|
||||
|
||||
void dispatchCombineHistory() {
|
||||
combineHistoryTask.dispatch();
|
||||
PausableDispatchTask currentTask;
|
||||
synchronized (this) {
|
||||
currentTask = combineHistoryTask;
|
||||
}
|
||||
if (currentTask != null) {
|
||||
currentTask.dispatch();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
calculateScheduler.shutdownNow();
|
||||
combineHistoryScheduler.shutdownNow();
|
||||
public synchronized void destroy() {
|
||||
started = false;
|
||||
if (calculateTask != null) {
|
||||
calculateTask.cancel();
|
||||
}
|
||||
if (combineHistoryTask != null) {
|
||||
combineHistoryTask.cancel();
|
||||
}
|
||||
if (calculateScheduler != null) {
|
||||
calculateScheduler.shutdownNow();
|
||||
calculateScheduler = null;
|
||||
}
|
||||
if (combineHistoryScheduler != null) {
|
||||
combineHistoryScheduler.shutdownNow();
|
||||
combineHistoryScheduler = null;
|
||||
}
|
||||
if (calculateExecutor != null) {
|
||||
calculateExecutor.shutdownNow();
|
||||
calculateExecutor = null;
|
||||
}
|
||||
if (combineHistoryExecutor != null) {
|
||||
combineHistoryExecutor.shutdownNow();
|
||||
combineHistoryExecutor = null;
|
||||
}
|
||||
calculateTask = null;
|
||||
combineHistoryTask = null;
|
||||
}
|
||||
|
||||
private void runCalculate() {
|
||||
@@ -319,70 +438,4 @@ public class CalculateStatus implements DisposableBean {
|
||||
})
|
||||
.factory());
|
||||
}
|
||||
|
||||
private static final class ScheduledDispatchTask {
|
||||
|
||||
private final ExecutorService executorService;
|
||||
private final Runnable task;
|
||||
private final Object lock = new Object();
|
||||
private boolean running;
|
||||
private int pendingRuns;
|
||||
|
||||
private ScheduledDispatchTask(ExecutorService executorService, Runnable task) {
|
||||
this.executorService = executorService;
|
||||
this.task = task;
|
||||
}
|
||||
|
||||
private void dispatch() {
|
||||
if (executorService == null) {
|
||||
task.run();
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (running) {
|
||||
pendingRuns++;
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
}
|
||||
submit();
|
||||
}
|
||||
|
||||
private void submit() {
|
||||
boolean submitted = false;
|
||||
try {
|
||||
executorService.execute(() -> {
|
||||
try {
|
||||
task.run();
|
||||
} finally {
|
||||
onComplete();
|
||||
}
|
||||
});
|
||||
submitted = true;
|
||||
} finally {
|
||||
if (!submitted) {
|
||||
synchronized (lock) {
|
||||
running = false;
|
||||
pendingRuns = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void onComplete() {
|
||||
boolean shouldRunAgain;
|
||||
synchronized (lock) {
|
||||
if (pendingRuns > 0) {
|
||||
pendingRuns--;
|
||||
shouldRunAgain = true;
|
||||
} else {
|
||||
running = false;
|
||||
shouldRunAgain = false;
|
||||
}
|
||||
}
|
||||
if (shouldRunAgain) {
|
||||
submit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.component.status;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.boot.CommandLineRunner;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Starts status-page calculation and history consolidation only in normal runtime. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class CalculateStatusLifecycle implements CommandLineRunner {
|
||||
|
||||
private final CalculateStatus calculateStatus;
|
||||
|
||||
public CalculateStatusLifecycle(CalculateStatus calculateStatus) {
|
||||
this.calculateStatus = calculateStatus;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(String... args) {
|
||||
calculateStatus.start();
|
||||
}
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.component.status;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.apache.hertzbeat.manager.maintenance.MaintenanceDeadline;
|
||||
import org.apache.hertzbeat.manager.maintenance.MetadataMaintenanceException;
|
||||
|
||||
/** Single-flight dispatch with pause, drain, and one coalesced due run. */
|
||||
final class PausableDispatchTask {
|
||||
|
||||
private final ExecutorService executorService;
|
||||
private final Runnable task;
|
||||
private final Object lock = new Object();
|
||||
private boolean running;
|
||||
private boolean pendingRun;
|
||||
private boolean paused;
|
||||
private boolean missedWhilePaused;
|
||||
private boolean cancelled;
|
||||
|
||||
PausableDispatchTask(ExecutorService executorService, Runnable task) {
|
||||
this.executorService = executorService;
|
||||
this.task = task;
|
||||
}
|
||||
|
||||
void dispatch() {
|
||||
synchronized (lock) {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
if (paused) {
|
||||
missedWhilePaused = true;
|
||||
return;
|
||||
}
|
||||
if (running) {
|
||||
pendingRun = true;
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
}
|
||||
runOrSubmit();
|
||||
}
|
||||
|
||||
void pauseAdmission() {
|
||||
synchronized (lock) {
|
||||
paused = true;
|
||||
missedWhilePaused |= pendingRun;
|
||||
pendingRun = false;
|
||||
}
|
||||
}
|
||||
|
||||
void awaitDrained(MaintenanceDeadline deadline) {
|
||||
synchronized (lock) {
|
||||
while (running) {
|
||||
long remainingNanos = deadline.remainingNanos();
|
||||
if (remainingNanos <= 0) {
|
||||
throw MetadataMaintenanceException.quiesceTimeout();
|
||||
}
|
||||
try {
|
||||
TimeUnit.NANOSECONDS.timedWait(lock, remainingNanos);
|
||||
} catch (InterruptedException exception) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw MetadataMaintenanceException.quiesceInterrupted();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void resumeAdmission() {
|
||||
boolean shouldRun = false;
|
||||
synchronized (lock) {
|
||||
if (!paused) {
|
||||
return;
|
||||
}
|
||||
paused = false;
|
||||
if (missedWhilePaused) {
|
||||
missedWhilePaused = false;
|
||||
if (running) {
|
||||
pendingRun = true;
|
||||
} else if (!cancelled) {
|
||||
running = true;
|
||||
shouldRun = true;
|
||||
}
|
||||
}
|
||||
lock.notifyAll();
|
||||
}
|
||||
if (shouldRun) {
|
||||
runOrSubmit();
|
||||
}
|
||||
}
|
||||
|
||||
void cancel() {
|
||||
synchronized (lock) {
|
||||
cancelled = true;
|
||||
paused = true;
|
||||
pendingRun = false;
|
||||
missedWhilePaused = false;
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
private void runOrSubmit() {
|
||||
if (executorService == null) {
|
||||
try {
|
||||
task.run();
|
||||
} finally {
|
||||
onComplete();
|
||||
}
|
||||
return;
|
||||
}
|
||||
boolean submitted = false;
|
||||
try {
|
||||
executorService.execute(() -> {
|
||||
try {
|
||||
task.run();
|
||||
} finally {
|
||||
onComplete();
|
||||
}
|
||||
});
|
||||
submitted = true;
|
||||
} finally {
|
||||
if (!submitted) {
|
||||
synchronized (lock) {
|
||||
running = false;
|
||||
pendingRun = false;
|
||||
lock.notifyAll();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void onComplete() {
|
||||
boolean shouldRunAgain;
|
||||
synchronized (lock) {
|
||||
if (cancelled || paused) {
|
||||
running = false;
|
||||
pendingRun = false;
|
||||
shouldRunAgain = false;
|
||||
} else if (pendingRun) {
|
||||
pendingRun = false;
|
||||
shouldRunAgain = true;
|
||||
} else {
|
||||
running = false;
|
||||
shouldRunAgain = false;
|
||||
}
|
||||
lock.notifyAll();
|
||||
}
|
||||
if (shouldRunAgain) {
|
||||
runOrSubmit();
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
-4
@@ -35,9 +35,9 @@ import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
|
||||
import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
|
||||
import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.manager.service.AccountService;
|
||||
import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl;
|
||||
import org.jspecify.annotations.NonNull;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.stereotype.Component;
|
||||
@@ -103,7 +103,7 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
|
||||
}
|
||||
touchTokenLastUsedTime(token);
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("Managed token validation failed");
|
||||
log.warn("Managed token validation failed ({})", e.getClass().getSimpleName());
|
||||
return writeError(response, HttpStatus.SERVICE_UNAVAILABLE, TOKEN_VALIDATION_UNAVAILABLE);
|
||||
}
|
||||
}
|
||||
@@ -135,7 +135,8 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
|
||||
if (rejectReason != null) {
|
||||
return rejectReason;
|
||||
}
|
||||
rejectReason = accountService.checkManagedTokenAccess(getCurrentUserId(subject), extractClaimedRoles(subject));
|
||||
rejectReason = accountService.checkManagedTokenAccess(
|
||||
getCurrentUserId(subject), extractClaimedRoles(subject), extractCredentialVersion(subject));
|
||||
if (rejectReason != null) {
|
||||
return rejectReason;
|
||||
}
|
||||
@@ -155,7 +156,7 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
|
||||
if (principalMap == null) {
|
||||
return false;
|
||||
}
|
||||
Object managed = principalMap.getPrincipal(AccountServiceImpl.CLAIM_MANAGED);
|
||||
Object managed = principalMap.getPrincipal(ObservabilityAccessTokenGateway.CLAIM_MANAGED);
|
||||
return managed instanceof Boolean ? (Boolean) managed : Boolean.parseBoolean(String.valueOf(managed));
|
||||
}
|
||||
|
||||
@@ -181,6 +182,16 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
|
||||
return principal == null ? null : String.valueOf(principal);
|
||||
}
|
||||
|
||||
private Long extractCredentialVersion(SubjectSum subject) {
|
||||
PrincipalMap principalMap = subject.getPrincipalMap();
|
||||
if (principalMap == null) {
|
||||
return null;
|
||||
}
|
||||
Object claimedVersion = principalMap.getPrincipal(
|
||||
ObservabilityAccessTokenGateway.CLAIM_CREDENTIAL_VERSION);
|
||||
return claimedVersion instanceof Number number ? number.longValue() : null;
|
||||
}
|
||||
|
||||
private String bindManagedCollectorBoundary(HttpServletRequest request, SubjectSum subject) {
|
||||
PrincipalMap principalMap = subject.getPrincipalMap();
|
||||
if (principalMap == null || !AuthTokenScopes.MANAGED_COLLECTOR_AUDIENCE.equals(
|
||||
|
||||
+2
@@ -22,6 +22,7 @@ import jakarta.annotation.Resource;
|
||||
import java.security.SecureRandom;
|
||||
import lombok.SneakyThrows;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.common.util.AesUtil;
|
||||
import org.apache.hertzbeat.manager.pojo.dto.MuteConfig;
|
||||
import org.apache.hertzbeat.manager.pojo.dto.SystemSecret;
|
||||
@@ -42,6 +43,7 @@ import org.springframework.stereotype.Component;
|
||||
*/
|
||||
@Component
|
||||
@Order(value = Ordered.HIGHEST_PRECEDENCE + 2)
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public class ConfigInitializer implements SmartLifecycle {
|
||||
|
||||
private boolean running = false;
|
||||
|
||||
@@ -104,4 +104,11 @@ public interface AuthTokenDao extends JpaRepository<AuthToken, Long>, JpaSpecifi
|
||||
@Transactional
|
||||
@Query("UPDATE AuthToken t SET t.lastUsedTime = :lastUsedTime WHERE t.tokenHash = :tokenHash")
|
||||
void updateLastUsedTime(@Param("tokenHash") String tokenHash, @Param("lastUsedTime") LocalDateTime lastUsedTime);
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query("UPDATE AuthToken t SET t.status = 1, t.revokedBy = :revokedBy, t.revokedTime = :revokedTime "
|
||||
+ "WHERE t.creator = :creator AND t.status = 0")
|
||||
int revokeActiveByCreator(@Param("creator") String creator, @Param("revokedBy") String revokedBy,
|
||||
@Param("revokedTime") LocalDateTime revokedTime);
|
||||
}
|
||||
|
||||
+38
-4
@@ -21,12 +21,16 @@ import static org.apache.hertzbeat.common.constants.CommonConstants.COLLECTOR_ST
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayState;
|
||||
import org.apache.hertzbeat.common.entity.dto.ManagedOtelRuntimeStatus.OtlpGatewayTransport;
|
||||
import org.apache.hertzbeat.common.entity.manager.Collector;
|
||||
import org.apache.hertzbeat.common.support.exception.CommonException;
|
||||
import org.apache.hertzbeat.manager.dao.CollectorDao;
|
||||
import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake;
|
||||
import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.ErrorCode;
|
||||
import org.apache.hertzbeat.manager.pojo.dto.CollectorInstrumentationIntake.Gateway;
|
||||
import org.apache.hertzbeat.manager.scheduler.runtime.CollectorRuntimeStatusRegistry;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
@@ -37,14 +41,22 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver
|
||||
|
||||
private final CollectorDao collectorDao;
|
||||
private final CollectorIntakeAdvertisementCodec codec;
|
||||
private final CollectorRuntimeStatusRegistry runtimeStatuses;
|
||||
|
||||
public CollectorIntakeAdvertisementService(CollectorDao collectorDao, CollectorIntakeAdvertisementCodec codec) {
|
||||
public CollectorIntakeAdvertisementService(
|
||||
CollectorDao collectorDao,
|
||||
CollectorIntakeAdvertisementCodec codec,
|
||||
CollectorRuntimeStatusRegistry runtimeStatuses) {
|
||||
this.collectorDao = collectorDao;
|
||||
this.codec = codec;
|
||||
this.runtimeStatuses = runtimeStatuses;
|
||||
}
|
||||
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public CollectorInstrumentationIntake update(String collectorName, CollectorIntakeAdvertisementRequest request) {
|
||||
if (request.gateway() != Gateway.COLLECTOR) {
|
||||
throw new IllegalArgumentException("Collector intake advertisement must be Collector-owned");
|
||||
}
|
||||
Collector collector = requireCollector(collectorName);
|
||||
collector.setInstrumentationIntake(codec.encode(request));
|
||||
collectorDao.save(collector);
|
||||
@@ -73,13 +85,35 @@ public class CollectorIntakeAdvertisementService implements CollectorIntakeAdver
|
||||
return CollectorInstrumentationIntake.unavailable(
|
||||
collectorId, ErrorCode.INTAKE_ADVERTISEMENT_INVALID);
|
||||
}
|
||||
if (request.gateway() == Gateway.COLLECTOR && collector.getStatus() != COLLECTOR_STATUS_ONLINE) {
|
||||
return CollectorInstrumentationIntake.unavailable(
|
||||
collectorId, ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE);
|
||||
if (request.gateway() == Gateway.COLLECTOR) {
|
||||
if (collector.getStatus() != COLLECTOR_STATUS_ONLINE || !runtimeSupports(collectorId, request)) {
|
||||
return CollectorInstrumentationIntake.unavailable(
|
||||
collectorId, ErrorCode.INTAKE_ADVERTISEMENT_UNAVAILABLE);
|
||||
}
|
||||
}
|
||||
return request.available(collectorId);
|
||||
}
|
||||
|
||||
private boolean runtimeSupports(String collectorId, CollectorIntakeAdvertisementRequest request) {
|
||||
return runtimeStatuses.current(collectorId)
|
||||
.map(reported -> supports(reported.status(), request))
|
||||
.orElse(false);
|
||||
}
|
||||
|
||||
private boolean supports(ManagedOtelRuntimeStatus status, CollectorIntakeAdvertisementRequest request) {
|
||||
if (!status.enabled()
|
||||
|| status.state() != ManagedOtelRuntimeStatus.RuntimeState.RUNNING
|
||||
|| status.otlpGateway().state() != OtlpGatewayState.AVAILABLE) {
|
||||
return false;
|
||||
}
|
||||
return request.capabilities().stream().allMatch(capability -> switch (capability) {
|
||||
case OTLP_HTTP_PROTOBUF -> status.otlpGateway().supportedTransports()
|
||||
.contains(OtlpGatewayTransport.HTTP_PROTOBUF);
|
||||
case OTLP_GRPC -> status.otlpGateway().supportedTransports()
|
||||
.contains(OtlpGatewayTransport.GRPC);
|
||||
});
|
||||
}
|
||||
|
||||
private Collector requireCollector(String collectorName) {
|
||||
return collectorDao.findCollectorByName(collectorName)
|
||||
.orElseThrow(() -> new CommonException("Collector not found: " + collectorName));
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmGroupReduce;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Sequential cut across alert producers whose downstream path writes management metadata. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Order(400)
|
||||
public final class AlertMetadataMaintenanceParticipant implements MetadataMaintenanceParticipant {
|
||||
|
||||
private final PeriodicAlertRuleScheduler periodicScheduler;
|
||||
private final AlarmCommonReduce commonReduce;
|
||||
private final AlarmGroupReduce groupReduce;
|
||||
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
|
||||
private boolean periodicPaused;
|
||||
private boolean commonPaused;
|
||||
private boolean groupPaused;
|
||||
|
||||
public AlertMetadataMaintenanceParticipant(
|
||||
PeriodicAlertRuleScheduler periodicScheduler,
|
||||
AlarmCommonReduce commonReduce,
|
||||
AlarmGroupReduce groupReduce) {
|
||||
this.periodicScheduler = periodicScheduler;
|
||||
this.commonReduce = commonReduce;
|
||||
this.groupReduce = groupReduce;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String participantId() {
|
||||
return "alert-control-metadata";
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void quiesce(Duration timeout) {
|
||||
if (phase == MetadataMaintenancePhase.QUIESCED) {
|
||||
return;
|
||||
}
|
||||
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
|
||||
phase = MetadataMaintenancePhase.QUIESCING;
|
||||
try {
|
||||
periodicScheduler.pauseAdmission();
|
||||
periodicPaused = true;
|
||||
periodicScheduler.awaitDrained(deadline.remainingNanos());
|
||||
commonReduce.pauseAdmission();
|
||||
commonPaused = true;
|
||||
commonReduce.awaitDrained(deadline.remainingNanos());
|
||||
groupReduce.pauseAdmission();
|
||||
groupPaused = true;
|
||||
groupReduce.awaitDrained(deadline.remainingNanos());
|
||||
phase = MetadataMaintenancePhase.QUIESCED;
|
||||
} catch (InterruptedException exception) {
|
||||
resumePausedStages();
|
||||
Thread.currentThread().interrupt();
|
||||
throw MetadataMaintenanceException.quiesceInterrupted();
|
||||
} catch (TimeoutException exception) {
|
||||
resumePausedStages();
|
||||
throw MetadataMaintenanceException.quiesceTimeout();
|
||||
} catch (RuntimeException exception) {
|
||||
resumePausedStages();
|
||||
throw MetadataMaintenanceException.participantFailure();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void resume() {
|
||||
if (phase == MetadataMaintenancePhase.RUNNING) {
|
||||
return;
|
||||
}
|
||||
if (!resumePausedStages()) {
|
||||
throw MetadataMaintenanceException.resumeFailure();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean resumePausedStages() {
|
||||
boolean resumed = true;
|
||||
if (groupPaused) {
|
||||
try {
|
||||
groupReduce.resumeAdmission();
|
||||
groupPaused = false;
|
||||
} catch (RuntimeException exception) {
|
||||
resumed = false;
|
||||
}
|
||||
}
|
||||
if (commonPaused) {
|
||||
try {
|
||||
commonReduce.resumeAdmission();
|
||||
commonPaused = false;
|
||||
} catch (RuntimeException exception) {
|
||||
resumed = false;
|
||||
}
|
||||
}
|
||||
if (periodicPaused) {
|
||||
try {
|
||||
periodicScheduler.resumeAdmission();
|
||||
periodicPaused = false;
|
||||
} catch (RuntimeException exception) {
|
||||
resumed = false;
|
||||
}
|
||||
}
|
||||
phase = resumed ? MetadataMaintenancePhase.RUNNING : MetadataMaintenancePhase.RECOVERY_REQUIRED;
|
||||
return resumed;
|
||||
}
|
||||
}
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import org.apache.hertzbeat.alert.calculate.CollectorAlertHandler;
|
||||
import org.apache.hertzbeat.common.concurrent.WorkAdmissionGate;
|
||||
import org.apache.hertzbeat.common.entity.dto.CollectorInfo;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.manager.scheduler.CollectorJobScheduler;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Coalesces collector lifecycle intent while draining metadata and paired alert work. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@Order(300)
|
||||
public final class CollectorLifecycleMaintenanceParticipant implements MetadataMaintenanceParticipant {
|
||||
|
||||
private final Object lock = new Object();
|
||||
private final CollectorJobScheduler scheduler;
|
||||
private final CollectorAlertHandler alertHandler;
|
||||
private final WorkAdmissionGate maintenanceGate = new WorkAdmissionGate();
|
||||
private final Map<String, Transition> pendingTransitions = new HashMap<>();
|
||||
private final Set<String> runningIdentities = new HashSet<>();
|
||||
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
|
||||
private long generation;
|
||||
|
||||
public CollectorLifecycleMaintenanceParticipant(
|
||||
CollectorJobScheduler scheduler, CollectorAlertHandler alertHandler) {
|
||||
this.scheduler = scheduler;
|
||||
this.alertHandler = alertHandler;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String participantId() {
|
||||
return "collector-control-metadata";
|
||||
}
|
||||
|
||||
public void collectorOnline(String identity, CollectorInfo collectorInfo, boolean submitAlert) {
|
||||
submit(identity, true, collectorInfo, submitAlert);
|
||||
}
|
||||
|
||||
public void collectorOffline(String identity, boolean submitAlert) {
|
||||
submit(identity, false, null, submitAlert);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void quiesce(Duration timeout) {
|
||||
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
|
||||
synchronized (lock) {
|
||||
if (phase == MetadataMaintenancePhase.QUIESCED) {
|
||||
return;
|
||||
}
|
||||
phase = MetadataMaintenancePhase.QUIESCING;
|
||||
maintenanceGate.pauseAdmission();
|
||||
}
|
||||
try {
|
||||
maintenanceGate.awaitDrained(deadline.remainingNanos());
|
||||
synchronized (lock) {
|
||||
phase = MetadataMaintenancePhase.QUIESCED;
|
||||
}
|
||||
} catch (InterruptedException exception) {
|
||||
reopenAfterFailedQuiesce();
|
||||
Thread.currentThread().interrupt();
|
||||
throw MetadataMaintenanceException.quiesceInterrupted();
|
||||
} catch (TimeoutException exception) {
|
||||
reopenAfterFailedQuiesce();
|
||||
throw MetadataMaintenanceException.quiesceTimeout();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void resume() {
|
||||
while (true) {
|
||||
Transition transition;
|
||||
synchronized (lock) {
|
||||
transition = pendingTransitions.values().stream()
|
||||
.min(Comparator.comparingLong(Transition::generation))
|
||||
.orElse(null);
|
||||
if (transition == null) {
|
||||
maintenanceGate.resumeAdmission();
|
||||
phase = MetadataMaintenancePhase.RUNNING;
|
||||
return;
|
||||
}
|
||||
pendingTransitions.remove(transition.identity(), transition);
|
||||
runningIdentities.add(transition.identity());
|
||||
}
|
||||
try {
|
||||
execute(transition);
|
||||
} catch (RuntimeException exception) {
|
||||
synchronized (lock) {
|
||||
pendingTransitions.putIfAbsent(transition.identity(), transition);
|
||||
runningIdentities.remove(transition.identity());
|
||||
}
|
||||
throw exception;
|
||||
} finally {
|
||||
synchronized (lock) {
|
||||
runningIdentities.remove(transition.identity());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void submit(String identity, boolean online, CollectorInfo collectorInfo, boolean submitAlert) {
|
||||
WorkAdmissionGate.Permit permit;
|
||||
Transition transition;
|
||||
synchronized (lock) {
|
||||
transition = new Transition(identity, online, collectorInfo, submitAlert, ++generation);
|
||||
if (runningIdentities.contains(identity)) {
|
||||
pendingTransitions.put(identity, transition);
|
||||
return;
|
||||
}
|
||||
// A failed transition is retained only until a newer observed intent supersedes it.
|
||||
pendingTransitions.remove(identity);
|
||||
permit = maintenanceGate.tryAcquire();
|
||||
if (permit == null) {
|
||||
pendingTransitions.put(identity, transition);
|
||||
return;
|
||||
}
|
||||
runningIdentities.add(identity);
|
||||
}
|
||||
executeAdmitted(transition, permit);
|
||||
}
|
||||
|
||||
private void executeAdmitted(Transition firstTransition, WorkAdmissionGate.Permit firstPermit) {
|
||||
Transition transition = firstTransition;
|
||||
WorkAdmissionGate.Permit permit = firstPermit;
|
||||
RuntimeException firstFailure = null;
|
||||
while (true) {
|
||||
boolean transitionFailed = false;
|
||||
try {
|
||||
execute(transition);
|
||||
} catch (RuntimeException exception) {
|
||||
transitionFailed = true;
|
||||
if (firstFailure == null) {
|
||||
firstFailure = exception;
|
||||
} else if (firstFailure.getSuppressed().length == 0) {
|
||||
firstFailure.addSuppressed(exception);
|
||||
}
|
||||
} finally {
|
||||
permit.close();
|
||||
}
|
||||
synchronized (lock) {
|
||||
runningIdentities.remove(transition.identity());
|
||||
Transition next = pendingTransitions.remove(transition.identity());
|
||||
if (next == null) {
|
||||
if (transitionFailed) {
|
||||
pendingTransitions.put(transition.identity(), transition);
|
||||
}
|
||||
throwIfFailed(firstFailure);
|
||||
return;
|
||||
}
|
||||
permit = maintenanceGate.tryAcquire();
|
||||
if (permit == null) {
|
||||
pendingTransitions.put(next.identity(), next);
|
||||
throwIfFailed(firstFailure);
|
||||
return;
|
||||
}
|
||||
runningIdentities.add(next.identity());
|
||||
transition = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void throwIfFailed(RuntimeException failure) {
|
||||
if (failure != null) {
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
private void execute(Transition transition) {
|
||||
if (transition.online()) {
|
||||
if (transition.submitAlert()) {
|
||||
alertHandler.online(transition.identity());
|
||||
}
|
||||
scheduler.collectorGoOnline(transition.identity(), transition.collectorInfo());
|
||||
return;
|
||||
}
|
||||
scheduler.collectorGoOffline(transition.identity());
|
||||
if (transition.submitAlert()) {
|
||||
alertHandler.offline(transition.identity());
|
||||
}
|
||||
}
|
||||
|
||||
private void reopenAfterFailedQuiesce() {
|
||||
synchronized (lock) {
|
||||
maintenanceGate.resumeAdmission();
|
||||
phase = MetadataMaintenancePhase.RUNNING;
|
||||
}
|
||||
}
|
||||
|
||||
private record Transition(
|
||||
String identity,
|
||||
boolean online,
|
||||
CollectorInfo collectorInfo,
|
||||
boolean submitAlert,
|
||||
long generation) {
|
||||
}
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease;
|
||||
|
||||
/** Releases one acquired maintenance window in strict reverse order with retryable progress. */
|
||||
final class CompositeMigrationMaintenanceLease implements MigrationMaintenanceLease {
|
||||
|
||||
private final DeploymentSingletonLease authorityLease;
|
||||
private final MigrationSourceLease sourceLease;
|
||||
private final MetadataMaintenanceLease producerLease;
|
||||
private final MetadataWriteMaintenanceLease writeLease;
|
||||
private final Runnable reservationRelease;
|
||||
private boolean writeReleased;
|
||||
private boolean producerReleased;
|
||||
private boolean sourceReleased;
|
||||
private boolean authorityReleased;
|
||||
private boolean sourceCallbackActive;
|
||||
private Thread sourceCallbackOwner;
|
||||
|
||||
CompositeMigrationMaintenanceLease(
|
||||
DeploymentSingletonLease authorityLease,
|
||||
MigrationSourceLease sourceLease,
|
||||
MetadataMaintenanceLease producerLease,
|
||||
MetadataWriteMaintenanceLease writeLease,
|
||||
Runnable reservationRelease) {
|
||||
this.authorityLease = authorityLease;
|
||||
this.sourceLease = sourceLease;
|
||||
this.producerLease = producerLease;
|
||||
this.writeLease = writeLease;
|
||||
this.reservationRelease = reservationRelease;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void withSourceConnection(MigrationSourceAction action) {
|
||||
if (action == null) {
|
||||
throw MigrationMaintenanceException.invalidRequest();
|
||||
}
|
||||
if (writeReleased || producerReleased || sourceReleased || authorityReleased) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
if (sourceCallbackActive) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
sourceCallbackActive = true;
|
||||
sourceCallbackOwner = Thread.currentThread();
|
||||
try {
|
||||
sourceLease.withConnection(action);
|
||||
} finally {
|
||||
sourceCallbackOwner = null;
|
||||
sourceCallbackActive = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
if (sourceCallbackActive && sourceCallbackOwner == Thread.currentThread()) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
try {
|
||||
releaseInOrder();
|
||||
} catch (MigrationMaintenanceException exception) {
|
||||
throw exception;
|
||||
} catch (RuntimeException exception) {
|
||||
throw MigrationMaintenanceException.resumeFailure();
|
||||
}
|
||||
reservationRelease.run();
|
||||
}
|
||||
|
||||
private void releaseInOrder() {
|
||||
if (!writeReleased) {
|
||||
writeLease.close();
|
||||
writeReleased = true;
|
||||
}
|
||||
if (!producerReleased) {
|
||||
producerLease.resume();
|
||||
producerReleased = true;
|
||||
}
|
||||
if (!sourceReleased) {
|
||||
sourceLease.close();
|
||||
sourceReleased = true;
|
||||
}
|
||||
if (!authorityReleased) {
|
||||
authorityLease.close();
|
||||
authorityReleased = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import javax.sql.DataSource;
|
||||
|
||||
/** Bounded JDBC connection acquisition that closes every result arriving after its deadline. */
|
||||
final class DeadlineConnectionAcquirer implements AutoCloseable {
|
||||
|
||||
private final DataSource dataSource;
|
||||
private final ThreadPoolExecutor executor;
|
||||
|
||||
DeadlineConnectionAcquirer(DataSource dataSource) {
|
||||
this.dataSource = dataSource;
|
||||
executor = new ThreadPoolExecutor(0, 1, 30, TimeUnit.SECONDS, new SynchronousQueue<>(),
|
||||
Thread.ofPlatform().daemon(true).name("migration-source-connection", 0).factory());
|
||||
}
|
||||
|
||||
Connection acquire(Duration timeout) {
|
||||
long timeoutNanos;
|
||||
try {
|
||||
timeoutNanos = timeout.toNanos();
|
||||
} catch (ArithmeticException exception) {
|
||||
throw MigrationMaintenanceException.invalidRequest();
|
||||
}
|
||||
Attempt attempt = new Attempt();
|
||||
try {
|
||||
executor.execute(() -> connect(attempt));
|
||||
} catch (RejectedExecutionException exception) {
|
||||
if (executor.isShutdown()) {
|
||||
throw MigrationMaintenanceException.sourceUnavailable();
|
||||
}
|
||||
throw MigrationMaintenanceException.timeout();
|
||||
}
|
||||
return attempt.await(timeoutNanos);
|
||||
}
|
||||
|
||||
private void connect(Attempt attempt) {
|
||||
Connection acquired = null;
|
||||
Throwable failure = null;
|
||||
try {
|
||||
acquired = dataSource.getConnection();
|
||||
} catch (Throwable connectionFailure) {
|
||||
failure = connectionFailure;
|
||||
}
|
||||
attempt.complete(acquired, failure);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
private static final class Attempt {
|
||||
|
||||
private final Object lock = new Object();
|
||||
private final CountDownLatch completed = new CountDownLatch(1);
|
||||
private Connection connection;
|
||||
private Throwable failure;
|
||||
private boolean abandoned;
|
||||
private boolean finished;
|
||||
|
||||
private Connection await(long timeoutNanos) {
|
||||
try {
|
||||
if (completed.await(timeoutNanos, TimeUnit.NANOSECONDS)) {
|
||||
return claimCompleted();
|
||||
}
|
||||
return abandonOrClaim();
|
||||
} catch (InterruptedException exception) {
|
||||
abandon();
|
||||
Thread.currentThread().interrupt();
|
||||
throw MigrationMaintenanceException.interrupted();
|
||||
}
|
||||
}
|
||||
|
||||
private void complete(Connection acquired, Throwable acquiredFailure) {
|
||||
synchronized (lock) {
|
||||
if (abandoned) {
|
||||
closeLate(acquired);
|
||||
} else {
|
||||
connection = acquired;
|
||||
failure = acquiredFailure;
|
||||
}
|
||||
finished = true;
|
||||
}
|
||||
completed.countDown();
|
||||
}
|
||||
|
||||
private Connection abandonOrClaim() {
|
||||
synchronized (lock) {
|
||||
if (finished) {
|
||||
return claimCompletedLocked();
|
||||
}
|
||||
abandoned = true;
|
||||
}
|
||||
throw MigrationMaintenanceException.timeout();
|
||||
}
|
||||
|
||||
private void abandon() {
|
||||
synchronized (lock) {
|
||||
if (!finished) {
|
||||
abandoned = true;
|
||||
} else {
|
||||
closeLate(connection);
|
||||
connection = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Connection claimCompleted() {
|
||||
synchronized (lock) {
|
||||
return claimCompletedLocked();
|
||||
}
|
||||
}
|
||||
|
||||
private Connection claimCompletedLocked() {
|
||||
if (failure instanceof Error error) {
|
||||
throw error;
|
||||
}
|
||||
if (failure != null || connection == null) {
|
||||
throw MigrationMaintenanceException.sourceUnavailable();
|
||||
}
|
||||
Connection claimed = connection;
|
||||
connection = null;
|
||||
return claimed;
|
||||
}
|
||||
|
||||
private static void closeLate(Connection lateConnection) {
|
||||
if (lateConnection == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
lateConnection.close();
|
||||
} catch (SQLException | RuntimeException exception) {
|
||||
// A late connection never becomes a lease; its details remain private.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.function.LongSupplier;
|
||||
import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionCoordinator;
|
||||
import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionErrorCode;
|
||||
import org.apache.hertzbeat.common.transaction.MetadataWriteAdmissionException;
|
||||
import org.apache.hertzbeat.common.transaction.MetadataWriteMaintenanceLease;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Composes deployment, source, producer, and transaction fences without owning their state machines. */
|
||||
@Component
|
||||
public final class DefaultMigrationMaintenanceOrchestrator implements MigrationMaintenanceOrchestrator {
|
||||
|
||||
private final ReentrantLock lock = new ReentrantLock();
|
||||
private final DeploymentSingletonAuthority deploymentAuthority;
|
||||
private final MigrationSourceGuard sourceGuard;
|
||||
private final MetadataMaintenanceCoordinator producerCoordinator;
|
||||
private final MetadataWriteAdmissionCoordinator writeCoordinator;
|
||||
private final LongSupplier ticker;
|
||||
private String operationId;
|
||||
private Object ownerToken;
|
||||
private MetadataMaintenanceLease recoveryProducerLease;
|
||||
private boolean recoveryProducerCoordinatorRequired;
|
||||
private MigrationSourceLease recoverySourceLease;
|
||||
private DeploymentSingletonLease recoveryAuthorityLease;
|
||||
|
||||
@Autowired
|
||||
public DefaultMigrationMaintenanceOrchestrator(
|
||||
DeploymentSingletonAuthority deploymentAuthority,
|
||||
MigrationSourceGuard sourceGuard,
|
||||
MetadataMaintenanceCoordinator producerCoordinator,
|
||||
MetadataWriteAdmissionCoordinator writeCoordinator) {
|
||||
this(deploymentAuthority, sourceGuard, producerCoordinator, writeCoordinator, System::nanoTime);
|
||||
}
|
||||
|
||||
DefaultMigrationMaintenanceOrchestrator(
|
||||
DeploymentSingletonAuthority deploymentAuthority,
|
||||
MigrationSourceGuard sourceGuard,
|
||||
MetadataMaintenanceCoordinator producerCoordinator,
|
||||
MetadataWriteAdmissionCoordinator writeCoordinator,
|
||||
LongSupplier ticker) {
|
||||
this.deploymentAuthority = deploymentAuthority;
|
||||
this.sourceGuard = sourceGuard;
|
||||
this.producerCoordinator = producerCoordinator;
|
||||
this.writeCoordinator = writeCoordinator;
|
||||
this.ticker = ticker;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MigrationMaintenanceLease acquire(String requestedOperationId, Duration timeout) {
|
||||
requireRequest(requestedOperationId);
|
||||
MaintenanceDeadline deadline;
|
||||
try {
|
||||
deadline = MaintenanceDeadline.start(timeout, ticker);
|
||||
} catch (MetadataMaintenanceException exception) {
|
||||
throw MigrationMaintenanceException.invalidRequest();
|
||||
}
|
||||
Object token = reserve(requestedOperationId);
|
||||
DeploymentSingletonLease authorityLease = null;
|
||||
MigrationSourceLease sourceLease = null;
|
||||
MetadataMaintenanceLease producerLease = null;
|
||||
try {
|
||||
authorityLease = deploymentAuthority.acquire(requestedOperationId, deadline.remaining());
|
||||
sourceLease = sourceGuard.fence(requestedOperationId, deadline.remaining());
|
||||
producerLease = producerCoordinator.quiesce(requestedOperationId, deadline.remaining());
|
||||
MetadataWriteMaintenanceLease writeLease =
|
||||
writeCoordinator.acquire(requestedOperationId, deadline.remaining());
|
||||
return new CompositeMigrationMaintenanceLease(
|
||||
authorityLease, sourceLease, producerLease, writeLease, () -> releaseReservation(token));
|
||||
} catch (Error error) {
|
||||
if (cleanupFailedAcquisition(error, producerLease, sourceLease, authorityLease)) {
|
||||
releaseReservation(token);
|
||||
} else {
|
||||
retainRecovery(producerLease, sourceLease, authorityLease);
|
||||
}
|
||||
throw error;
|
||||
} catch (RuntimeException exception) {
|
||||
MigrationMaintenanceException primary = mapFailure(exception);
|
||||
if (cleanupFailedAcquisition(primary, producerLease, sourceLease, authorityLease)) {
|
||||
releaseReservation(token);
|
||||
} else {
|
||||
retainRecovery(producerLease, sourceLease, authorityLease);
|
||||
}
|
||||
throw primary;
|
||||
}
|
||||
}
|
||||
|
||||
private Object reserve(String requestedOperationId) {
|
||||
lock.lock();
|
||||
try {
|
||||
if (ownerToken != null) {
|
||||
if (!requestedOperationId.equals(operationId)
|
||||
|| !hasRecovery()) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
recoverFailedAcquisition();
|
||||
}
|
||||
Object token = new Object();
|
||||
ownerToken = token;
|
||||
operationId = requestedOperationId;
|
||||
return token;
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean cleanupFailedAcquisition(
|
||||
Throwable primary,
|
||||
MetadataMaintenanceLease producerLease,
|
||||
MigrationSourceLease sourceLease,
|
||||
DeploymentSingletonLease authorityLease) {
|
||||
boolean interrupted = Thread.currentThread().isInterrupted();
|
||||
boolean producerReleased = producerCoordinator.snapshot().phase() == MetadataMaintenancePhase.RUNNING;
|
||||
if (producerLease != null) {
|
||||
producerReleased = suppressCleanup(primary, producerLease::resume);
|
||||
}
|
||||
boolean sourceReleased = sourceLease == null;
|
||||
if (producerReleased && sourceLease != null) {
|
||||
sourceReleased = suppressCleanup(primary, sourceLease::close);
|
||||
}
|
||||
boolean authorityReleased = authorityLease == null;
|
||||
if (producerReleased && sourceReleased && authorityLease != null) {
|
||||
authorityReleased = suppressCleanup(primary, authorityLease::close);
|
||||
}
|
||||
if (interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return producerReleased && sourceReleased && authorityReleased;
|
||||
}
|
||||
|
||||
private boolean suppressCleanup(Throwable primary, Runnable cleanup) {
|
||||
try {
|
||||
cleanup.run();
|
||||
return true;
|
||||
} catch (RuntimeException exception) {
|
||||
primary.addSuppressed(MigrationMaintenanceException.resumeFailure());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void retainRecovery(
|
||||
MetadataMaintenanceLease producerLease,
|
||||
MigrationSourceLease sourceLease,
|
||||
DeploymentSingletonLease authorityLease) {
|
||||
lock.lock();
|
||||
try {
|
||||
recoveryProducerLease = producerLease;
|
||||
recoveryProducerCoordinatorRequired = producerLease == null
|
||||
&& producerCoordinator.snapshot().phase() != MetadataMaintenancePhase.RUNNING;
|
||||
recoverySourceLease = sourceLease;
|
||||
recoveryAuthorityLease = authorityLease;
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void recoverFailedAcquisition() {
|
||||
try {
|
||||
if (recoveryProducerLease != null) {
|
||||
recoveryProducerLease.resume();
|
||||
recoveryProducerLease = null;
|
||||
}
|
||||
if (recoveryProducerCoordinatorRequired) {
|
||||
producerCoordinator.recover(operationId);
|
||||
recoveryProducerCoordinatorRequired = false;
|
||||
}
|
||||
if (recoverySourceLease != null) {
|
||||
recoverySourceLease.close();
|
||||
recoverySourceLease = null;
|
||||
}
|
||||
if (recoveryAuthorityLease != null) {
|
||||
recoveryAuthorityLease.close();
|
||||
recoveryAuthorityLease = null;
|
||||
}
|
||||
ownerToken = null;
|
||||
operationId = null;
|
||||
} catch (RuntimeException exception) {
|
||||
throw MigrationMaintenanceException.resumeFailure();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean hasRecovery() {
|
||||
return recoveryProducerLease != null
|
||||
|| recoveryProducerCoordinatorRequired
|
||||
|| recoverySourceLease != null
|
||||
|| recoveryAuthorityLease != null;
|
||||
}
|
||||
|
||||
private MigrationMaintenanceException mapFailure(RuntimeException exception) {
|
||||
if (exception instanceof MigrationMaintenanceException migrationFailure) {
|
||||
return migrationFailure;
|
||||
}
|
||||
if (exception instanceof MetadataMaintenanceException maintenanceFailure) {
|
||||
return switch (maintenanceFailure.code()) {
|
||||
case INVALID_REQUEST -> MigrationMaintenanceException.invalidRequest();
|
||||
case OPERATION_CONFLICT, STALE_LEASE -> MigrationMaintenanceException.operationConflict();
|
||||
case QUIESCE_TIMEOUT -> MigrationMaintenanceException.timeout();
|
||||
case QUIESCE_INTERRUPTED -> MigrationMaintenanceException.interrupted();
|
||||
case PARTICIPANT_FAILURE, RESUME_FAILURE -> MigrationMaintenanceException.maintenanceFailure();
|
||||
};
|
||||
}
|
||||
if (exception instanceof MetadataWriteAdmissionException writeFailure) {
|
||||
MetadataWriteAdmissionErrorCode code = writeFailure.code();
|
||||
return switch (code) {
|
||||
case INVALID_REQUEST -> MigrationMaintenanceException.invalidRequest();
|
||||
case OPERATION_CONFLICT, MAINTENANCE_ACTIVE -> MigrationMaintenanceException.operationConflict();
|
||||
case DRAIN_TIMEOUT -> MigrationMaintenanceException.timeout();
|
||||
case ACQUISITION_INTERRUPTED -> MigrationMaintenanceException.interrupted();
|
||||
};
|
||||
}
|
||||
return MigrationMaintenanceException.maintenanceFailure();
|
||||
}
|
||||
|
||||
private void releaseReservation(Object token) {
|
||||
lock.lock();
|
||||
try {
|
||||
if (ownerToken == token) {
|
||||
ownerToken = null;
|
||||
operationId = null;
|
||||
recoveryProducerLease = null;
|
||||
recoveryProducerCoordinatorRequired = false;
|
||||
recoverySourceLease = null;
|
||||
recoveryAuthorityLease = null;
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void requireRequest(String requestedOperationId) {
|
||||
if (requestedOperationId == null || requestedOperationId.isBlank()) {
|
||||
throw MigrationMaintenanceException.invalidRequest();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/** Proves and fences single-manager ownership for one logical deployment. */
|
||||
public interface DeploymentSingletonAuthority {
|
||||
|
||||
DeploymentSingletonLease acquire(String operationId, Duration timeout);
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Owner capability for one authoritative deployment-singleton fence. */
|
||||
public interface DeploymentSingletonLease extends AutoCloseable {
|
||||
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Locale;
|
||||
|
||||
/** Pure embedded-H2 source-safety classification. */
|
||||
final class EmbeddedH2SourceClassifier {
|
||||
|
||||
private static final String H2_PREFIX = "jdbc:h2:";
|
||||
|
||||
private EmbeddedH2SourceClassifier() {
|
||||
}
|
||||
|
||||
static boolean isSafeEmbeddedSource(String productName, String jdbcUrl) {
|
||||
if (productName == null || !"h2".equals(productName.trim().toLowerCase(Locale.ROOT))
|
||||
|| jdbcUrl == null) {
|
||||
return false;
|
||||
}
|
||||
String url = jdbcUrl.trim().toLowerCase(Locale.ROOT);
|
||||
if (!url.startsWith(H2_PREFIX) || hasUnsafeSetting(url)) {
|
||||
return false;
|
||||
}
|
||||
String location = url.substring(H2_PREFIX.length());
|
||||
if (location.startsWith("mem:")) {
|
||||
return !isRemote(location.substring("mem:".length()));
|
||||
}
|
||||
String fileLocation = location.startsWith("file:")
|
||||
? location.substring("file:".length())
|
||||
: location;
|
||||
return !isRemote(fileLocation) && (location.startsWith("file:")
|
||||
|| location.startsWith("./")
|
||||
|| location.startsWith("../")
|
||||
|| location.startsWith(".\\")
|
||||
|| location.startsWith("..\\")
|
||||
|| location.startsWith("~/")
|
||||
|| location.startsWith("/")
|
||||
|| isWindowsDrive(location));
|
||||
}
|
||||
|
||||
static boolean matchesConfiguredSource(String configuredUrl, String actualUrl) {
|
||||
String configuredLocation = sourceLocation(configuredUrl);
|
||||
String actualLocation = sourceLocation(actualUrl);
|
||||
if (configuredLocation == null || actualLocation == null) {
|
||||
return false;
|
||||
}
|
||||
if (configuredLocation.startsWith("mem:") || actualLocation.startsWith("mem:")) {
|
||||
return configuredLocation.equals(actualLocation);
|
||||
}
|
||||
try {
|
||||
return localPath(configuredLocation).equals(localPath(actualLocation));
|
||||
} catch (RuntimeException exception) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static String sourceLocation(String jdbcUrl) {
|
||||
if (jdbcUrl == null || !jdbcUrl.regionMatches(true, 0, H2_PREFIX, 0, H2_PREFIX.length())) {
|
||||
return null;
|
||||
}
|
||||
String location = jdbcUrl.substring(H2_PREFIX.length()).split(";", 2)[0];
|
||||
return location.regionMatches(true, 0, "file:", 0, "file:".length())
|
||||
? location.substring("file:".length()) : location;
|
||||
}
|
||||
|
||||
private static Path localPath(String location) {
|
||||
String expanded = location.startsWith("~/") || location.startsWith("~\\")
|
||||
? System.getProperty("user.home") + location.substring(1) : location;
|
||||
return Path.of(expanded).toAbsolutePath().normalize();
|
||||
}
|
||||
|
||||
private static boolean isRemote(String location) {
|
||||
return location.contains("://") || location.startsWith("//") || location.startsWith("\\\\");
|
||||
}
|
||||
|
||||
private static boolean isWindowsDrive(String location) {
|
||||
return location.length() >= 3
|
||||
&& Character.isLetter(location.charAt(0))
|
||||
&& location.charAt(1) == ':'
|
||||
&& (location.charAt(2) == '\\' || location.charAt(2) == '/');
|
||||
}
|
||||
|
||||
private static boolean hasUnsafeSetting(String url) {
|
||||
for (String setting : url.split(";")) {
|
||||
String normalized = setting.strip();
|
||||
if (normalized.startsWith("auto_server") || normalized.equals("file_lock=no")) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.time.Duration;
|
||||
import javax.sql.DataSource;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.boot.jdbc.autoconfigure.DataSourceProperties;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Holds an embedded H2 connection after proving its configured local access mode is safe. */
|
||||
@Component
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
public final class EmbeddedH2SourceGuard implements MigrationSourceGuard, DisposableBean {
|
||||
|
||||
private final DataSourceProperties dataSourceProperties;
|
||||
private final DeadlineConnectionAcquirer connectionAcquirer;
|
||||
|
||||
public EmbeddedH2SourceGuard(DataSource dataSource, DataSourceProperties dataSourceProperties) {
|
||||
this.dataSourceProperties = dataSourceProperties;
|
||||
this.connectionAcquirer = new DeadlineConnectionAcquirer(dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public MigrationSourceLease fence(String operationId, Duration timeout) {
|
||||
requireRequest(operationId, timeout);
|
||||
String configuredUrl = dataSourceProperties.getUrl();
|
||||
if (!EmbeddedH2SourceClassifier.isSafeEmbeddedSource("H2", configuredUrl)) {
|
||||
throw MigrationMaintenanceException.sourceUnavailable();
|
||||
}
|
||||
Connection connection = connectionAcquirer.acquire(timeout);
|
||||
try {
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
String productName = metadata.getDatabaseProductName();
|
||||
String actualUrl = metadata.getURL();
|
||||
if (!EmbeddedH2SourceClassifier.isSafeEmbeddedSource(productName, actualUrl)
|
||||
|| !EmbeddedH2SourceClassifier.matchesConfiguredSource(configuredUrl, actualUrl)) {
|
||||
closeRejectedConnection(connection);
|
||||
throw MigrationMaintenanceException.sourceUnavailable();
|
||||
}
|
||||
return new ConnectionSourceLease(connection);
|
||||
} catch (MigrationMaintenanceException exception) {
|
||||
throw exception;
|
||||
} catch (SQLException | RuntimeException exception) {
|
||||
closeRejectedConnection(connection);
|
||||
throw MigrationMaintenanceException.sourceUnavailable();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
connectionAcquirer.close();
|
||||
}
|
||||
|
||||
private void requireRequest(String operationId, Duration timeout) {
|
||||
if (operationId == null || operationId.isBlank() || timeout == null || timeout.isNegative()) {
|
||||
throw MigrationMaintenanceException.invalidRequest();
|
||||
}
|
||||
}
|
||||
|
||||
private void closeRejectedConnection(Connection connection) {
|
||||
try {
|
||||
connection.close();
|
||||
} catch (SQLException | RuntimeException exception) {
|
||||
// The stable source failure remains primary.
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ConnectionSourceLease implements MigrationSourceLease {
|
||||
|
||||
private final Connection connection;
|
||||
private boolean closed;
|
||||
private boolean callbackActive;
|
||||
private Thread callbackOwner;
|
||||
|
||||
private ConnectionSourceLease(Connection connection) {
|
||||
this.connection = connection;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void withConnection(MigrationSourceAction action) {
|
||||
if (action == null) {
|
||||
throw MigrationMaintenanceException.invalidRequest();
|
||||
}
|
||||
if (closed) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
if (callbackActive) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
callbackActive = true;
|
||||
callbackOwner = Thread.currentThread();
|
||||
try {
|
||||
action.execute(connection);
|
||||
} finally {
|
||||
callbackOwner = null;
|
||||
callbackActive = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void close() {
|
||||
if (callbackActive && callbackOwner == Thread.currentThread()) {
|
||||
throw MigrationMaintenanceException.operationConflict();
|
||||
}
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
connection.close();
|
||||
closed = true;
|
||||
} catch (SQLException | RuntimeException exception) {
|
||||
throw MigrationMaintenanceException.resumeFailure();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Re-reads durable installation convergence for each migration fence. */
|
||||
@FunctionalInterface
|
||||
public interface InstallationConvergenceVerifier {
|
||||
|
||||
boolean isFullyConverged();
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.function.LongSupplier;
|
||||
|
||||
/** Shared monotonic deadline for one process-local maintenance transition. */
|
||||
public final class MaintenanceDeadline {
|
||||
|
||||
private final long timeoutNanos;
|
||||
private final long startedNanos;
|
||||
private final LongSupplier ticker;
|
||||
|
||||
private MaintenanceDeadline(long timeoutNanos, long startedNanos, LongSupplier ticker) {
|
||||
this.timeoutNanos = timeoutNanos;
|
||||
this.startedNanos = startedNanos;
|
||||
this.ticker = ticker;
|
||||
}
|
||||
|
||||
public static MaintenanceDeadline start(Duration timeout) {
|
||||
return start(timeout, System::nanoTime);
|
||||
}
|
||||
|
||||
static MaintenanceDeadline start(Duration timeout, LongSupplier ticker) {
|
||||
if (timeout == null || timeout.isNegative()) {
|
||||
throw MetadataMaintenanceException.invalidRequest();
|
||||
}
|
||||
try {
|
||||
long timeoutNanos = timeout.toNanos();
|
||||
return new MaintenanceDeadline(timeoutNanos, ticker.getAsLong(), ticker);
|
||||
} catch (ArithmeticException exception) {
|
||||
throw MetadataMaintenanceException.invalidRequest();
|
||||
}
|
||||
}
|
||||
|
||||
public long remainingNanos() {
|
||||
long elapsedNanos = ticker.getAsLong() - startedNanos;
|
||||
if (elapsedNanos <= 0) {
|
||||
return timeoutNanos;
|
||||
}
|
||||
if (elapsedNanos >= timeoutNanos) {
|
||||
return 0;
|
||||
}
|
||||
return timeoutNanos - elapsedNanos;
|
||||
}
|
||||
|
||||
public Duration remaining() {
|
||||
return Duration.ofNanos(remainingNanos());
|
||||
}
|
||||
}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Coordinates process-local metadata producers without controlling their shared executors.
|
||||
*
|
||||
* <p>A future migration workflow must quiesce this coordinator before acquiring metadata write
|
||||
* admission. On exit it must release write admission before resuming this lease. Keeping those
|
||||
* capabilities separate prevents producer lifecycle from becoming a transaction or datasource
|
||||
* switch.</p>
|
||||
*/
|
||||
@Component
|
||||
public final class MetadataMaintenanceCoordinator {
|
||||
|
||||
private final ReentrantLock lock = new ReentrantLock();
|
||||
private final List<MetadataMaintenanceParticipant> participants;
|
||||
private MetadataMaintenancePhase phase = MetadataMaintenancePhase.RUNNING;
|
||||
private String operationId;
|
||||
private long epoch;
|
||||
private Object leaseToken;
|
||||
|
||||
public MetadataMaintenanceCoordinator(List<MetadataMaintenanceParticipant> participants) {
|
||||
this.participants = List.copyOf(participants);
|
||||
validateParticipants(this.participants);
|
||||
}
|
||||
|
||||
/** Pause producers in registration order and drain work admitted before the pause. */
|
||||
public MetadataMaintenanceLease quiesce(String requestedOperationId, Duration timeout) {
|
||||
MaintenanceDeadline deadline = MaintenanceDeadline.start(timeout);
|
||||
requireOperationId(requestedOperationId);
|
||||
Acquisition acquisition = beginAcquisition(requestedOperationId);
|
||||
|
||||
List<MetadataMaintenanceParticipant> started = new ArrayList<>(participants.size());
|
||||
try {
|
||||
for (MetadataMaintenanceParticipant participant : participants) {
|
||||
started.add(participant);
|
||||
participant.quiesce(deadline.remaining());
|
||||
}
|
||||
} catch (MetadataMaintenanceException exception) {
|
||||
rollback(acquisition, started);
|
||||
throw exception;
|
||||
} catch (Error error) {
|
||||
rollback(acquisition, started);
|
||||
throw error;
|
||||
} catch (RuntimeException exception) {
|
||||
rollback(acquisition, started);
|
||||
throw MetadataMaintenanceException.participantFailure();
|
||||
}
|
||||
return completeAcquisition(acquisition);
|
||||
}
|
||||
|
||||
public MetadataMaintenanceSnapshot snapshot() {
|
||||
lock.lock();
|
||||
try {
|
||||
return new MetadataMaintenanceSnapshot(phase, operationId, epoch);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void recover(String requestedOperationId) {
|
||||
requireOperationId(requestedOperationId);
|
||||
lock.lock();
|
||||
try {
|
||||
if (phase != MetadataMaintenancePhase.RECOVERY_REQUIRED
|
||||
|| !requestedOperationId.equals(operationId)) {
|
||||
throw MetadataMaintenanceException.operationConflict();
|
||||
}
|
||||
if (!resumeAllParticipants()) {
|
||||
throw MetadataMaintenanceException.resumeFailure();
|
||||
}
|
||||
reopen();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void resume(String resumedOperationId, long resumedEpoch, Object resumedToken) {
|
||||
lock.lock();
|
||||
try {
|
||||
if (!ownsResumeLease(resumedOperationId, resumedEpoch, resumedToken)) {
|
||||
throw MetadataMaintenanceException.staleLease();
|
||||
}
|
||||
phase = MetadataMaintenancePhase.QUIESCING;
|
||||
boolean failed = false;
|
||||
for (int index = participants.size() - 1; index >= 0; index--) {
|
||||
try {
|
||||
participants.get(index).resume();
|
||||
} catch (RuntimeException exception) {
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
if (failed) {
|
||||
throw MetadataMaintenanceException.resumeFailure();
|
||||
}
|
||||
reopen();
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private Acquisition beginAcquisition(String requestedOperationId) {
|
||||
lock.lock();
|
||||
try {
|
||||
if (phase == MetadataMaintenancePhase.RECOVERY_REQUIRED) {
|
||||
if (!requestedOperationId.equals(operationId)) {
|
||||
throw MetadataMaintenanceException.operationConflict();
|
||||
}
|
||||
if (!resumeAllParticipants()) {
|
||||
throw MetadataMaintenanceException.resumeFailure();
|
||||
}
|
||||
reopen();
|
||||
}
|
||||
if (phase != MetadataMaintenancePhase.RUNNING) {
|
||||
throw MetadataMaintenanceException.operationConflict();
|
||||
}
|
||||
phase = MetadataMaintenancePhase.QUIESCING;
|
||||
operationId = requestedOperationId;
|
||||
long requestedEpoch = ++epoch;
|
||||
Object requestedToken = new Object();
|
||||
leaseToken = requestedToken;
|
||||
return new Acquisition(requestedOperationId, requestedEpoch, requestedToken);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean resumeAllParticipants() {
|
||||
boolean resumed = true;
|
||||
for (int index = participants.size() - 1; index >= 0; index--) {
|
||||
try {
|
||||
participants.get(index).resume();
|
||||
} catch (RuntimeException exception) {
|
||||
resumed = false;
|
||||
}
|
||||
}
|
||||
return resumed;
|
||||
}
|
||||
|
||||
private MetadataMaintenanceLease completeAcquisition(Acquisition acquisition) {
|
||||
lock.lock();
|
||||
try {
|
||||
if (!ownsAcquisition(acquisition)) {
|
||||
throw MetadataMaintenanceException.operationConflict();
|
||||
}
|
||||
phase = MetadataMaintenancePhase.QUIESCED;
|
||||
return lease(acquisition);
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private void rollback(Acquisition acquisition, List<MetadataMaintenanceParticipant> started) {
|
||||
Collections.reverse(started);
|
||||
boolean failed = false;
|
||||
for (MetadataMaintenanceParticipant participant : started) {
|
||||
try {
|
||||
participant.resume();
|
||||
} catch (RuntimeException exception) {
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
lock.lock();
|
||||
try {
|
||||
if (ownsAcquisition(acquisition) && !failed) {
|
||||
reopen();
|
||||
} else if (ownsAcquisition(acquisition)) {
|
||||
phase = MetadataMaintenancePhase.RECOVERY_REQUIRED;
|
||||
}
|
||||
} finally {
|
||||
lock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean ownsAcquisition(Acquisition acquisition) {
|
||||
return phase == MetadataMaintenancePhase.QUIESCING
|
||||
&& epoch == acquisition.epoch()
|
||||
&& operationId.equals(acquisition.operationId())
|
||||
&& leaseToken == acquisition.token();
|
||||
}
|
||||
|
||||
private boolean ownsResumeLease(String resumedOperationId, long resumedEpoch, Object resumedToken) {
|
||||
return (phase == MetadataMaintenancePhase.QUIESCED
|
||||
|| phase == MetadataMaintenancePhase.QUIESCING)
|
||||
&& epoch == resumedEpoch
|
||||
&& operationId.equals(resumedOperationId)
|
||||
&& leaseToken == resumedToken;
|
||||
}
|
||||
|
||||
private MetadataMaintenanceLease lease(Acquisition acquisition) {
|
||||
return new MetadataMaintenanceLease(
|
||||
this, acquisition.operationId(), acquisition.epoch(), acquisition.token());
|
||||
}
|
||||
|
||||
private void reopen() {
|
||||
phase = MetadataMaintenancePhase.RUNNING;
|
||||
operationId = null;
|
||||
leaseToken = null;
|
||||
}
|
||||
|
||||
private void requireOperationId(String requestedOperationId) {
|
||||
if (requestedOperationId == null || requestedOperationId.isBlank()) {
|
||||
throw MetadataMaintenanceException.invalidRequest();
|
||||
}
|
||||
}
|
||||
|
||||
private void validateParticipants(List<MetadataMaintenanceParticipant> registeredParticipants) {
|
||||
Set<String> participantIds = new HashSet<>(registeredParticipants.size());
|
||||
for (MetadataMaintenanceParticipant participant : registeredParticipants) {
|
||||
String participantId = participant.participantId();
|
||||
if (participantId == null || participantId.isBlank() || !participantIds.add(participantId)) {
|
||||
throw MetadataMaintenanceException.invalidRequest();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private record Acquisition(String operationId, long epoch, Object token) {
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Stable, secret-free control-plane maintenance failure classifications. */
|
||||
public enum MetadataMaintenanceErrorCode {
|
||||
INVALID_REQUEST("invalid_request"),
|
||||
OPERATION_CONFLICT("operation_conflict"),
|
||||
QUIESCE_TIMEOUT("quiesce_timeout"),
|
||||
QUIESCE_INTERRUPTED("quiesce_interrupted"),
|
||||
PARTICIPANT_FAILURE("participant_failure"),
|
||||
RESUME_FAILURE("resume_failure"),
|
||||
STALE_LEASE("stale_lease");
|
||||
|
||||
private final String wireCode;
|
||||
|
||||
MetadataMaintenanceErrorCode(String wireCode) {
|
||||
this.wireCode = wireCode;
|
||||
}
|
||||
|
||||
public String wireCode() {
|
||||
return wireCode;
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Safe maintenance failure that never exposes participant work or persistence details. */
|
||||
public final class MetadataMaintenanceException extends RuntimeException {
|
||||
|
||||
private static final String INVALID_MESSAGE = "Metadata maintenance request is invalid";
|
||||
private static final String CONFLICT_MESSAGE = "Metadata maintenance operation is already active";
|
||||
private static final String TIMEOUT_MESSAGE = "Metadata producer drain timed out";
|
||||
private static final String INTERRUPTED_MESSAGE = "Metadata producer drain was interrupted";
|
||||
private static final String PARTICIPANT_MESSAGE = "Metadata producer could not be paused";
|
||||
private static final String RESUME_MESSAGE = "Metadata producer could not be resumed";
|
||||
private static final String STALE_MESSAGE = "Metadata maintenance lease is stale";
|
||||
|
||||
private final MetadataMaintenanceErrorCode code;
|
||||
|
||||
private MetadataMaintenanceException(MetadataMaintenanceErrorCode code, String message) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public MetadataMaintenanceErrorCode code() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public String safeMessage() {
|
||||
return getMessage();
|
||||
}
|
||||
|
||||
static MetadataMaintenanceException invalidRequest() {
|
||||
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.INVALID_REQUEST, INVALID_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataMaintenanceException operationConflict() {
|
||||
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.OPERATION_CONFLICT, CONFLICT_MESSAGE);
|
||||
}
|
||||
|
||||
public static MetadataMaintenanceException quiesceTimeout() {
|
||||
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.QUIESCE_TIMEOUT, TIMEOUT_MESSAGE);
|
||||
}
|
||||
|
||||
public static MetadataMaintenanceException quiesceInterrupted() {
|
||||
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.QUIESCE_INTERRUPTED, INTERRUPTED_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataMaintenanceException participantFailure() {
|
||||
return new MetadataMaintenanceException(
|
||||
MetadataMaintenanceErrorCode.PARTICIPANT_FAILURE, PARTICIPANT_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataMaintenanceException resumeFailure() {
|
||||
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.RESUME_FAILURE, RESUME_MESSAGE);
|
||||
}
|
||||
|
||||
static MetadataMaintenanceException staleLease() {
|
||||
return new MetadataMaintenanceException(MetadataMaintenanceErrorCode.STALE_LEASE, STALE_MESSAGE);
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Epoch-bound capability that resumes metadata producers exactly once. */
|
||||
public final class MetadataMaintenanceLease implements AutoCloseable {
|
||||
|
||||
private final MetadataMaintenanceCoordinator coordinator;
|
||||
private final String operationId;
|
||||
private final long epoch;
|
||||
private final Object token;
|
||||
private boolean resumed;
|
||||
|
||||
MetadataMaintenanceLease(
|
||||
MetadataMaintenanceCoordinator coordinator, String operationId, long epoch, Object token) {
|
||||
this.coordinator = coordinator;
|
||||
this.operationId = operationId;
|
||||
this.epoch = epoch;
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
public synchronized void resume() {
|
||||
if (resumed) {
|
||||
return;
|
||||
}
|
||||
coordinator.resume(operationId, epoch, token);
|
||||
resumed = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
resume();
|
||||
}
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/** One metadata producer that can stop admitting work and drain already admitted work. */
|
||||
public interface MetadataMaintenanceParticipant {
|
||||
|
||||
String participantId();
|
||||
|
||||
/** Stop admitting work and drain work admitted before this call. */
|
||||
void quiesce(Duration timeout);
|
||||
|
||||
/** Resume normal admission without creating another scheduler or consumer. */
|
||||
void resume();
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Lifecycle of process-local metadata producer admission. */
|
||||
public enum MetadataMaintenancePhase {
|
||||
RUNNING,
|
||||
QUIESCING,
|
||||
QUIESCED,
|
||||
RECOVERY_REQUIRED
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Read-only process-local maintenance state without the lease capability. */
|
||||
public record MetadataMaintenanceSnapshot(
|
||||
MetadataMaintenancePhase phase, String operationId, long epoch) {
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import org.apache.hertzbeat.common.runtime.BusinessRuntimeGate;
|
||||
import org.apache.hertzbeat.common.runtime.ConditionalOnNormalBusinessRuntime;
|
||||
import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
/** Fail-closed fallbacks for maintenance facts that the runtime cannot prove. */
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
public class MigrationGuardConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnNormalBusinessRuntime
|
||||
@ConditionalOnBean(StandaloneDeploymentOwnerView.class)
|
||||
@ConditionalOnMissingBean(InstallationConvergenceVerifier.class)
|
||||
InstallationConvergenceVerifier normalInstallationConvergenceVerifier(
|
||||
InstallationRecordRepository records, StandaloneDeploymentOwnerView owner) {
|
||||
return new NormalInstallationConvergenceVerifier(records, owner);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnBean({StandaloneDeploymentOwnerView.class, InstallationConvergenceVerifier.class})
|
||||
@ConditionalOnMissingBean(DeploymentSingletonAuthority.class)
|
||||
DeploymentSingletonAuthority deploymentSingletonAuthority(
|
||||
BusinessRuntimeGate runtimeGate,
|
||||
StandaloneDeploymentOwnerView owner,
|
||||
InstallationConvergenceVerifier convergence) {
|
||||
if (runtimeGate.isOpen()) {
|
||||
return new StandaloneDeploymentSingletonAuthority(owner, convergence);
|
||||
}
|
||||
return unavailableDeploymentSingletonAuthority();
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(DeploymentSingletonAuthority.class)
|
||||
DeploymentSingletonAuthority unavailableDeploymentSingletonAuthority() {
|
||||
return (operationId, timeout) -> {
|
||||
throw MigrationMaintenanceException.deploymentAuthorityUnavailable();
|
||||
};
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean(MigrationSourceGuard.class)
|
||||
MigrationSourceGuard unavailableMigrationSourceGuard() {
|
||||
return (operationId, timeout) -> {
|
||||
throw MigrationMaintenanceException.sourceUnavailable();
|
||||
};
|
||||
}
|
||||
}
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Stable safe failure categories for migration maintenance acquisition and release. */
|
||||
public enum MigrationMaintenanceErrorCode {
|
||||
MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE,
|
||||
MIGRATION_SOURCE_UNAVAILABLE,
|
||||
MIGRATION_MULTI_NODE_UNSUPPORTED,
|
||||
MIGRATION_OPERATION_CONFLICT,
|
||||
MIGRATION_MAINTENANCE_TIMEOUT,
|
||||
MIGRATION_MAINTENANCE_INTERRUPTED,
|
||||
MIGRATION_MAINTENANCE_FAILURE,
|
||||
MIGRATION_RESUME_FAILURE,
|
||||
INVALID_REQUEST
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Secret-free migration maintenance failure. */
|
||||
public final class MigrationMaintenanceException extends RuntimeException {
|
||||
|
||||
private final MigrationMaintenanceErrorCode code;
|
||||
|
||||
private MigrationMaintenanceException(MigrationMaintenanceErrorCode code, String message) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
}
|
||||
|
||||
public MigrationMaintenanceErrorCode code() {
|
||||
return code;
|
||||
}
|
||||
|
||||
public String safeMessage() {
|
||||
return getMessage();
|
||||
}
|
||||
|
||||
public static MigrationMaintenanceException deploymentAuthorityUnavailable() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_DEPLOYMENT_AUTHORITY_UNAVAILABLE,
|
||||
"Migration deployment authority is unavailable");
|
||||
}
|
||||
|
||||
public static MigrationMaintenanceException sourceUnavailable() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_SOURCE_UNAVAILABLE,
|
||||
"Migration metadata source is unavailable");
|
||||
}
|
||||
|
||||
public static MigrationMaintenanceException multiNodeUnsupported() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_MULTI_NODE_UNSUPPORTED,
|
||||
"Multi-node metadata migration is unsupported");
|
||||
}
|
||||
|
||||
public static MigrationMaintenanceException operationConflict() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_OPERATION_CONFLICT,
|
||||
"Migration maintenance operation is already active");
|
||||
}
|
||||
|
||||
static MigrationMaintenanceException timeout() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_TIMEOUT,
|
||||
"Migration maintenance acquisition timed out");
|
||||
}
|
||||
|
||||
static MigrationMaintenanceException interrupted() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_INTERRUPTED,
|
||||
"Migration maintenance acquisition was interrupted");
|
||||
}
|
||||
|
||||
public static MigrationMaintenanceException maintenanceFailure() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_MAINTENANCE_FAILURE,
|
||||
"Migration maintenance acquisition failed");
|
||||
}
|
||||
|
||||
static MigrationMaintenanceException resumeFailure() {
|
||||
return failure(MigrationMaintenanceErrorCode.MIGRATION_RESUME_FAILURE,
|
||||
"Migration maintenance release failed");
|
||||
}
|
||||
|
||||
public static MigrationMaintenanceException invalidRequest() {
|
||||
return failure(MigrationMaintenanceErrorCode.INVALID_REQUEST,
|
||||
"Migration maintenance request is invalid");
|
||||
}
|
||||
|
||||
private static MigrationMaintenanceException failure(
|
||||
MigrationMaintenanceErrorCode code, String message) {
|
||||
return new MigrationMaintenanceException(code, message);
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Owner capability for one fully acquired migration maintenance window. */
|
||||
public interface MigrationMaintenanceLease extends AutoCloseable {
|
||||
|
||||
/** Runs synchronous work against the exact source fenced by this maintenance window. */
|
||||
void withSourceConnection(MigrationSourceAction action);
|
||||
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/** Acquires the complete process-local maintenance window required before metadata migration. */
|
||||
public interface MigrationMaintenanceOrchestrator {
|
||||
|
||||
MigrationMaintenanceLease acquire(String operationId, Duration timeout);
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.sql.Connection;
|
||||
|
||||
/** Synchronous work scoped to the exact metadata source held by a maintenance lease. */
|
||||
@FunctionalInterface
|
||||
public interface MigrationSourceAction {
|
||||
|
||||
/**
|
||||
* Uses the guarded source only for this callback. The action must not retain, replace, or
|
||||
* independently close the connection. Only the bounded JDBC migration executor may invalidate
|
||||
* it on a fail-closed timeout or unknown-outcome path; final ownership remains with the lease.
|
||||
*/
|
||||
void execute(Connection source);
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
/** Fences a metadata source whose local access mode is safe for migration. */
|
||||
public interface MigrationSourceGuard {
|
||||
|
||||
MigrationSourceLease fence(String operationId, Duration timeout);
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
/** Capability that releases one safe metadata-source fence. */
|
||||
public interface MigrationSourceLease extends AutoCloseable {
|
||||
|
||||
/** Runs synchronous work against the exact source owned by this lease. */
|
||||
void withConnection(MigrationSourceAction action);
|
||||
|
||||
@Override
|
||||
void close();
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import org.apache.hertzbeat.manager.setup.installation.InstallationConvergenceService;
|
||||
import org.apache.hertzbeat.manager.setup.installation.InstallationMode;
|
||||
import org.apache.hertzbeat.manager.setup.installation.InstallationRecordRepository;
|
||||
|
||||
/** Normal-context adapter that re-reads fingerprint and database installation state. */
|
||||
public final class NormalInstallationConvergenceVerifier implements InstallationConvergenceVerifier {
|
||||
|
||||
private final InstallationRecordRepository records;
|
||||
private final StandaloneDeploymentOwnerView owner;
|
||||
|
||||
public NormalInstallationConvergenceVerifier(
|
||||
InstallationRecordRepository records, StandaloneDeploymentOwnerView owner) {
|
||||
this.records = records;
|
||||
this.owner = owner;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isFullyConverged() {
|
||||
Path root = owner.installationRoot();
|
||||
return new InstallationConvergenceService(
|
||||
records, root, root.resolve("data/config/.installation-fingerprint")).classify()
|
||||
== InstallationMode.FULL;
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.manager.maintenance;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
/** Non-owning view of the process-level standalone deployment lock. */
|
||||
public interface StandaloneDeploymentOwnerView {
|
||||
|
||||
Path installationRoot();
|
||||
|
||||
boolean isValid();
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user