mirror of
https://github.com/dromara/hertzbeat.git
synced 2026-09-17 09:40:58 +00:00
[alerter]fix: address group alert review gaps
This commit is contained in:
+4
@@ -151,6 +151,9 @@ final class DbAlertStoreHandlerImpl implements AlertStoreHandler {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void refreshGroupStatus(GroupAlert groupAlert) {
|
private void refreshGroupStatus(GroupAlert groupAlert) {
|
||||||
|
if (!CommonConstants.ALERT_STATUS_RESOLVED.equals(groupAlert.getStatus())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
List<String> alertFingerprints = groupAlert.getAlertFingerprints();
|
List<String> alertFingerprints = groupAlert.getAlertFingerprints();
|
||||||
if (alertFingerprints == null || alertFingerprints.isEmpty()) {
|
if (alertFingerprints == null || alertFingerprints.isEmpty()) {
|
||||||
return;
|
return;
|
||||||
@@ -164,5 +167,6 @@ final class DbAlertStoreHandlerImpl implements AlertStoreHandler {
|
|||||||
groupAlert.setStatus(hasFiringAlert
|
groupAlert.setStatus(hasFiringAlert
|
||||||
? CommonConstants.ALERT_STATUS_FIRING
|
? CommonConstants.ALERT_STATUS_FIRING
|
||||||
: CommonConstants.ALERT_STATUS_RESOLVED);
|
: CommonConstants.ALERT_STATUS_RESOLVED);
|
||||||
|
groupAlert.setAlerts(alerts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -291,7 +291,7 @@ public class AlarmGroupReduce implements DisposableBean {
|
|||||||
// For firing alerts, check repeat interval
|
// For firing alerts, check repeat interval
|
||||||
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
|
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
|
||||||
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
|
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
|
||||||
long repeatInterval = ruleConfig.getRepeatInterval() != null
|
long repeatInterval = ruleConfig != null && ruleConfig.getRepeatInterval() != null
|
||||||
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
|
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
|
||||||
// Skip if within repeat interval. The throttle only suppresses repeated firing
|
// Skip if within repeat interval. The throttle only suppresses repeated firing
|
||||||
// notifications; it must never swallow a pending resolved transition, so we still
|
// notifications; it must never swallow a pending resolved transition, so we still
|
||||||
|
|||||||
+21
@@ -183,6 +183,27 @@ class DbAlertStoreHandlerImplTest {
|
|||||||
|
|
||||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus());
|
assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus());
|
||||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, groupAlert.getStatus());
|
assertEquals(CommonConstants.ALERT_STATUS_FIRING, groupAlert.getStatus());
|
||||||
|
assertEquals(2, savedGroupAlert.getAlerts().size());
|
||||||
|
assertTrue(savedGroupAlert.getAlerts().stream()
|
||||||
|
.anyMatch(alert -> firingFingerprint.equals(alert.getFingerprint())
|
||||||
|
&& CommonConstants.ALERT_STATUS_FIRING.equals(alert.getStatus())));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void storeFiringGroupDoesNotQueryHistoricalAlerts() {
|
||||||
|
String groupKey = "instance:host1";
|
||||||
|
groupAlert.setGroupKey(groupKey);
|
||||||
|
groupAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
|
||||||
|
singleAlert.setFingerprint("cpu");
|
||||||
|
singleAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
|
||||||
|
when(groupAlertDao.findByGroupKey(groupKey)).thenReturn(null);
|
||||||
|
when(singleAlertDao.save(any(SingleAlert.class))).thenReturn(singleAlert);
|
||||||
|
when(groupAlertDao.save(any(GroupAlert.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||||
|
|
||||||
|
GroupAlert savedGroupAlert = dbAlertStoreHandler.store(groupAlert);
|
||||||
|
|
||||||
|
assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus());
|
||||||
|
verify(singleAlertDao, never()).findSingleAlertsByFingerprintIn(anyList());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+32
@@ -34,6 +34,8 @@ import java.util.Collections;
|
|||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
@@ -145,6 +147,25 @@ class AlarmGroupReduceTest {
|
|||||||
&& CommonConstants.ALERT_STATUS_RESOLVED.equals(alert.getStatus()))));
|
&& CommonConstants.ALERT_STATUS_RESOLVED.equals(alert.getStatus()))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void cachedGroupUsesDefaultRepeatIntervalWhenGroupRuleWasRemoved() throws Exception {
|
||||||
|
AlertGroupConverge rule = new AlertGroupConverge();
|
||||||
|
rule.setName("test-rule");
|
||||||
|
rule.setGroupLabels(Collections.singletonList("instance"));
|
||||||
|
alarmGroupReduce.refreshGroupDefines(Collections.singletonList(rule));
|
||||||
|
|
||||||
|
alarmGroupReduce.processGroupAlert(createAlert("cpu", CommonConstants.ALERT_STATUS_FIRING));
|
||||||
|
ageCachedGroupsPastDefaultWait();
|
||||||
|
alarmGroupReduce.refreshGroupDefines(Collections.emptyList());
|
||||||
|
|
||||||
|
alarmGroupReduce.dispatchCheckAndSendGroups();
|
||||||
|
|
||||||
|
verify(alarmInhibitReduce).inhibitAlarm(argThat(group ->
|
||||||
|
CommonConstants.ALERT_STATUS_FIRING.equals(group.getStatus())
|
||||||
|
&& group.getAlerts().size() == 1
|
||||||
|
&& group.getAlerts().get(0).getFingerprint().equals("cpu")));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception {
|
void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception {
|
||||||
CountDownLatch latch = new CountDownLatch(1);
|
CountDownLatch latch = new CountDownLatch(1);
|
||||||
@@ -279,6 +300,17 @@ class AlarmGroupReduceTest {
|
|||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void ageCachedGroupsPastDefaultWait() throws Exception {
|
||||||
|
Field groupCacheMapField = AlarmGroupReduce.class.getDeclaredField("groupCacheMap");
|
||||||
|
groupCacheMapField.setAccessible(true);
|
||||||
|
Map<?, ?> groupCacheMap = (Map<?, ?>) groupCacheMapField.get(alarmGroupReduce);
|
||||||
|
for (Object cache : groupCacheMap.values()) {
|
||||||
|
Method setCreateTime = cache.getClass().getDeclaredMethod("setCreateTime", long.class);
|
||||||
|
setCreateTime.setAccessible(true);
|
||||||
|
setCreateTime.invoke(cache, System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static final class TestAlarmGroupReduce extends AlarmGroupReduce {
|
private static final class TestAlarmGroupReduce extends AlarmGroupReduce {
|
||||||
|
|
||||||
private final CountDownLatch virtualThreadLatch;
|
private final CountDownLatch virtualThreadLatch;
|
||||||
|
|||||||
Reference in New Issue
Block a user