[alerter]fix: address group alert review gaps

This commit is contained in:
hutiefang
2026-08-30 23:09:02 +08:00
committed by hutiefang76
parent 9ac4608a8d
commit a9cae8b6ee
4 changed files with 58 additions and 1 deletions
@@ -151,6 +151,9 @@ final class DbAlertStoreHandlerImpl implements AlertStoreHandler {
}
private void refreshGroupStatus(GroupAlert groupAlert) {
if (!CommonConstants.ALERT_STATUS_RESOLVED.equals(groupAlert.getStatus())) {
return;
}
List<String> alertFingerprints = groupAlert.getAlertFingerprints();
if (alertFingerprints == null || alertFingerprints.isEmpty()) {
return;
@@ -164,5 +167,6 @@ final class DbAlertStoreHandlerImpl implements AlertStoreHandler {
groupAlert.setStatus(hasFiringAlert
? CommonConstants.ALERT_STATUS_FIRING
: CommonConstants.ALERT_STATUS_RESOLVED);
groupAlert.setAlerts(alerts);
}
}
@@ -291,7 +291,7 @@ public class AlarmGroupReduce implements DisposableBean {
// For firing alerts, check repeat interval
if (CommonConstants.ALERT_STATUS_FIRING.equals(status)) {
AlertGroupConverge ruleConfig = groupDefines.get(cache.getGroupDefineName());
long repeatInterval = ruleConfig.getRepeatInterval() != null
long repeatInterval = ruleConfig != null && ruleConfig.getRepeatInterval() != null
? ruleConfig.getRepeatInterval() * MS_PER_SECOND : DEFAULT_REPEAT_INTERVAL;
// Skip if within repeat interval. The throttle only suppresses repeated firing
// notifications; it must never swallow a pending resolved transition, so we still
@@ -183,6 +183,27 @@ class DbAlertStoreHandlerImplTest {
assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus());
assertEquals(CommonConstants.ALERT_STATUS_FIRING, groupAlert.getStatus());
assertEquals(2, savedGroupAlert.getAlerts().size());
assertTrue(savedGroupAlert.getAlerts().stream()
.anyMatch(alert -> firingFingerprint.equals(alert.getFingerprint())
&& CommonConstants.ALERT_STATUS_FIRING.equals(alert.getStatus())));
}
@Test
public void storeFiringGroupDoesNotQueryHistoricalAlerts() {
String groupKey = "instance:host1";
groupAlert.setGroupKey(groupKey);
groupAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
singleAlert.setFingerprint("cpu");
singleAlert.setStatus(CommonConstants.ALERT_STATUS_FIRING);
when(groupAlertDao.findByGroupKey(groupKey)).thenReturn(null);
when(singleAlertDao.save(any(SingleAlert.class))).thenReturn(singleAlert);
when(groupAlertDao.save(any(GroupAlert.class))).thenAnswer(invocation -> invocation.getArgument(0));
GroupAlert savedGroupAlert = dbAlertStoreHandler.store(groupAlert);
assertEquals(CommonConstants.ALERT_STATUS_FIRING, savedGroupAlert.getStatus());
verify(singleAlertDao, never()).findSingleAlertsByFingerprintIn(anyList());
}
}
@@ -34,6 +34,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -145,6 +147,25 @@ class AlarmGroupReduceTest {
&& CommonConstants.ALERT_STATUS_RESOLVED.equals(alert.getStatus()))));
}
@Test
void cachedGroupUsesDefaultRepeatIntervalWhenGroupRuleWasRemoved() throws Exception {
AlertGroupConverge rule = new AlertGroupConverge();
rule.setName("test-rule");
rule.setGroupLabels(Collections.singletonList("instance"));
alarmGroupReduce.refreshGroupDefines(Collections.singletonList(rule));
alarmGroupReduce.processGroupAlert(createAlert("cpu", CommonConstants.ALERT_STATUS_FIRING));
ageCachedGroupsPastDefaultWait();
alarmGroupReduce.refreshGroupDefines(Collections.emptyList());
alarmGroupReduce.dispatchCheckAndSendGroups();
verify(alarmInhibitReduce).inhibitAlarm(argThat(group ->
CommonConstants.ALERT_STATUS_FIRING.equals(group.getStatus())
&& group.getAlerts().size() == 1
&& group.getAlerts().get(0).getFingerprint().equals("cpu")));
}
@Test
void dispatchCheckAndSendGroupsRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
@@ -279,6 +300,17 @@ class AlarmGroupReduceTest {
.build();
}
private void ageCachedGroupsPastDefaultWait() throws Exception {
Field groupCacheMapField = AlarmGroupReduce.class.getDeclaredField("groupCacheMap");
groupCacheMapField.setAccessible(true);
Map<?, ?> groupCacheMap = (Map<?, ?>) groupCacheMapField.get(alarmGroupReduce);
for (Object cache : groupCacheMap.values()) {
Method setCreateTime = cache.getClass().getDeclaredMethod("setCreateTime", long.class);
setCreateTime.setAccessible(true);
setCreateTime.invoke(cache, System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(1));
}
}
private static final class TestAlarmGroupReduce extends AlarmGroupReduce {
private final CountDownLatch virtualThreadLatch;