wyl/hertzbeat
1
0
Fork 0
mirror of https://github.com/dromara/hertzbeat.git synced 2026-09-17 09:40:58 +00:00
Code Issues Projects Releases Wiki Activity

[alerter]bugfix: preserve group alert recovery events

Browse Source
This commit is contained in:
hutiefang76
2026-08-30 23:08:03 +08:00
parent 8e9f8da341
commit 9ac4608a8d
4 changed files with 105 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImpl.java
+17
View File
@@ -131,6 +131,7 @@ final class DbAlertStoreHandlerImpl implements AlertStoreHandler {
}
// Save alert group
groupAlert.setAlertFingerprints(alertFingerprints.stream().toList());
refreshGroupStatus(groupAlert);
GroupAlert savedGroupAlert = groupAlertDao.save(groupAlert);
savedGroupAlert.setAlerts(groupAlert.getAlerts());
return savedGroupAlert;
@@ -148,4 +149,20 @@ final class DbAlertStoreHandlerImpl implements AlertStoreHandler {
}
return locks;
}
private void refreshGroupStatus(GroupAlert groupAlert) {
List<String> alertFingerprints = groupAlert.getAlertFingerprints();
if (alertFingerprints == null || alertFingerprints.isEmpty()) {
return;
}
List<SingleAlert> alerts = singleAlertDao.findSingleAlertsByFingerprintIn(alertFingerprints);
if (alerts == null) {
return;
}
boolean hasFiringAlert = alerts.stream()
.anyMatch(alert -> CommonConstants.ALERT_STATUS_FIRING.equals(alert.getStatus()));
groupAlert.setStatus(hasFiringAlert
? CommonConstants.ALERT_STATUS_FIRING
: CommonConstants.ALERT_STATUS_RESOLVED);
}
}
hertzbeat-alerter/src/main/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduce.java
+1 -2
View File
@@ -293,7 +293,6 @@ public class AlarmGroupReduce implements DisposableBean {
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
long repeatInterval = ruleConfig.getRepeatInterval() != null
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
// Skip if within repeat interval. The throttle only suppresses repeated firing
// notifications; it must never swallow a pending resolved transition, so we still
// send when the batch carries a member that has just recovered.
@@ -402,7 +401,7 @@ public class AlarmGroupReduce implements DisposableBean {
.anyMatch(alert -> CommonConstants.ALERT_STATUS_FIRING.equals(alert.getStatus()))
? CommonConstants.ALERT_STATUS_FIRING : CommonConstants.ALERT_STATUS_RESOLVED;
}
@Data
private static class GroupAlertCache {
private String groupDefineName;
hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/notice/impl/DbAlertStoreHandlerImplTest.java
+46
View File
@@ -21,11 +21,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import org.apache.hertzbeat.alert.dao.GroupAlertDao;
import org.apache.hertzbeat.alert.dao.SingleAlertDao;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
import org.junit.jupiter.api.BeforeEach;
@@ -139,4 +141,48 @@ class DbAlertStoreHandlerImplTest {
assertTrue(locks.size() <= stripeCount);
}
@Test
public void storeResolvedAlertKeepsGroupFiringWhenOtherGroupAlertsStillFire() {
String groupKey = "instance:host1";
String resolvedFingerprint = "cpu";
String firingFingerprint = "memory";
groupAlert.setGroupKey(groupKey);
groupAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED);
singleAlert.setFingerprint(resolvedFingerprint);
singleAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED);
GroupAlert existingGroup = new GroupAlert();
existingGroup.setId(1L);
existingGroup.setAlertFingerprints(List.of(resolvedFingerprint, firingFingerprint));
when(groupAlertDao.findByGroupKey(groupKey)).thenReturn(existingGroup);
SingleAlert previousFiringAlert = new SingleAlert();
previousFiringAlert.setId(1L);
previousFiringAlert.setFingerprint(resolvedFingerprint);
previousFiringAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
previousFiringAlert.setStartAt(1000L);
previousFiringAlert.setActiveAt(2000L);
previousFiringAlert.setTriggerTimes(3);
when(singleAlertDao.findByFingerprint(resolvedFingerprint)).thenReturn(previousFiringAlert);
SingleAlert savedResolvedAlert = new SingleAlert();
savedResolvedAlert.setId(1L);
savedResolvedAlert.setFingerprint(resolvedFingerprint);
savedResolvedAlert.setStatus(CommonConstants.ALERT_STATUS_RESOLVED);
when(singleAlertDao.save(any(SingleAlert.class))).thenReturn(savedResolvedAlert);
SingleAlert existingFiringAlert = new SingleAlert();
existingFiringAlert.setFingerprint(firingFingerprint);
existingFiringAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
when(singleAlertDao.findSingleAlertsByFingerprintIn(anyList()))
.thenReturn(List.of(savedResolvedAlert, existingFiringAlert));
when(groupAlertDao.save(any(GroupAlert.class))).thenAnswer(invocation -> invocation.getArgument(0));
GroupAlert savedGroupAlert = dbAlertStoreHandler.store(groupAlert);
assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus());
assertEquals(CommonConstants.ALERT_STATUS_FIRING, groupAlert.getStatus());
}
}
hertzbeat-alerter/src/test/java/org/apache/hertzbeat/alert/reduce/AlarmGroupReduceTest.java
+41 -1
View File
@@ -26,6 +26,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.clearInvocations;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Arrays;
@@ -38,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.alert.dao.AlertGroupConvergeDao;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.entity.alerter.AlertGroupConverge;
import org.apache.hertzbeat.common.entity.alerter.GroupAlert;
@@ -68,7 +70,7 @@ class AlarmGroupReduceTest {
when(alertGroupConvergeDao.findAlertGroupConvergesByEnableIsTrue())
.thenReturn(Collections.emptyList());
alarmGroupReduce = new AlarmGroupReduce(alarmInhibitReduce, alertGroupConvergeDao,
new VirtualThreadProperties(), false);
new VirtualThreadProperties(false, null, null, null, null, null, null), false);
}
@AfterEach
@@ -114,6 +116,35 @@ class AlarmGroupReduceTest {
verify(alarmInhibitReduce, never()).inhibitAlarm(any()); // Should not send immediately due to group wait
}
@Test
void resolvedAlertInFiringGroupShouldBypassRepeatThrottle() {
AlertGroupConverge rule = new AlertGroupConverge();
rule.setName("test-rule");
rule.setGroupLabels(Collections.singletonList("instance"));
rule.setGroupWait(0L);
rule.setGroupInterval(0L);
rule.setRepeatInterval(60L);
alarmGroupReduce.refreshGroupDefines(Collections.singletonList(rule));
alarmGroupReduce.processGroupAlert(createAlert("cpu", CommonConstants.ALERT_STATUS_FIRING));
alarmGroupReduce.dispatchCheckAndSendGroups();
verify(alarmInhibitReduce).inhibitAlarm(argThat(group ->
CommonConstants.ALERT_STATUS_FIRING.equals(group.getStatus())
&& group.getAlerts().size() == 1
&& group.getAlerts().get(0).getFingerprint().equals("cpu")));
clearInvocations(alarmInhibitReduce);
alarmGroupReduce.processGroupAlert(createAlert("cpu", CommonConstants.ALERT_STATUS_FIRING));
alarmGroupReduce.processGroupAlert(createAlert("memory", CommonConstants.ALERT_STATUS_RESOLVED));
alarmGroupReduce.dispatchCheckAndSendGroups();
verify(alarmInhibitReduce).inhibitAlarm(argThat(group ->
CommonConstants.ALERT_STATUS_FIRING.equals(group.getStatus())
&& group.getAlerts().stream()
.anyMatch(alert -> "memory".equals(alert.getFingerprint())
&& CommonConstants.ALERT_STATUS_RESOLVED.equals(alert.getStatus()))));
}
@Test
void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
@@ -239,6 +270,15 @@ class AlarmGroupReduceTest {
return labels;
}
private SingleAlert createAlert(String fingerprint, String status) {
return SingleAlert.builder()
.fingerprint(fingerprint)
.status(status)
.labels(createLabels("instance", "host1"))
.annotations(Collections.emptyMap())
.build();
}
private static final class TestAlarmGroupReduce extends AlarmGroupReduce {
private final CountDownLatch virtualThreadLatch;